From 3734bc40952387c67b1f6a10a1fb6d059a920a93 Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Fri, 7 May 2021 09:38:47 -0400 Subject: [PATCH 01/15] This branch is based on common_gpu and should be merged only after common_gpu Add a new level_zero device (WIP) - copy device_cuda in device_level_zero and rename things - module_init and module_fini for level_zero Need to factorize a little bit more. Factorizing (need to do it in base) Port above new common --- CMakeLists.txt | 23 + cmake_modules/Findlevel-zero.cmake | 15 + parsec/include/parsec/parsec_options.h.in | 1 + parsec/mca/device/device.h | 3 +- .../device/level_zero/ValidateModule.CMake | 15 + .../mca/device/level_zero/device_level_zero.h | 87 + .../level_zero/device_level_zero_component.c | 341 +++ .../level_zero/device_level_zero_internal.h | 77 + .../level_zero/device_level_zero_module.c | 2667 +++++++++++++++++ parsec/mca/device/parsec_gpu.c | 97 + parsec/mca/device/transfer_gpu.c | 5 + 11 files changed, 3330 insertions(+), 1 deletion(-) create mode 100644 cmake_modules/Findlevel-zero.cmake create mode 100644 parsec/mca/device/level_zero/ValidateModule.CMake create mode 100644 parsec/mca/device/level_zero/device_level_zero.h create mode 100644 parsec/mca/device/level_zero/device_level_zero_component.c create mode 100644 parsec/mca/device/level_zero/device_level_zero_internal.h create mode 100644 parsec/mca/device/level_zero/device_level_zero_module.c create mode 100644 parsec/mca/device/parsec_gpu.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 2fc094ff7..ac7bc8ffc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,6 +167,12 @@ option(PARSEC_GPU_CUDA_ALLOC_PER_TILE "Tile based allocation engine for GPU memory (instead of internal management of a complete allocation)" OFF) mark_as_advanced(PARSEC_GPU_CUDA_ALLOC_PER_TILE) +option(PARSEC_GPU_WITH_LEVEL_ZERO + "Enable GPU support using LEVEL_ZERO kernels" ON) +option(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE + "Tile based allocation engine for GPU memory (instead of internal management + of a complete allocation)" OFF) +mark_as_advanced(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) option(PARSEC_GPU_WITH_OPENCL "Enable GPU support using OpenCL kernels" OFF) mark_as_advanced(PARSEC_GPU_WITH_OPENCL) # Hide this as it is not supported yet @@ -709,6 +715,16 @@ int main(int argc, char *argv[]) { set(PARSEC_HAVE_CU_COMPILER ${CMAKE_CUDA_COMPILER} CACHE BOOL "True if PaRSEC provide support for compiling .cu files") endif( PARSEC_GPU_WITH_CUDA ) + if( PARSEC_GPU_WITH_LEVEL_ZERO ) + find_package(level-zero) + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero") + if (LEVEL_ZERO_FOUND) + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/") + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero") + message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}") + endif (LEVEL_ZERO_FOUND) + endif( PARSEC_GPU_WITH_LEVEL_ZERO ) + find_package(AYUDAME QUIET) set(PARSEC_HAVE_AYUDAME ${AYUDAME_FOUND}) # @@ -886,6 +902,13 @@ include(AddDocumentedFiles) add_subdirectory(parsec) +# +# Add dependency to Level-Zero if it is enabled +# +if(PARSEC_HAVE_LEVEL_ZERO) + target_link_libraries(parsec PRIVATE level_zero::ze_loader) +endif(PARSEC_HAVE_LEVEL_ZERO) + # # Now continue with compiling the tests. # diff --git a/cmake_modules/Findlevel-zero.cmake b/cmake_modules/Findlevel-zero.cmake new file mode 100644 index 000000000..25ed408b7 --- /dev/null +++ b/cmake_modules/Findlevel-zero.cmake @@ -0,0 +1,15 @@ +find_package(PkgConfig QUIET) + +if(PKG_CONFIG_FOUND) + pkg_check_modules(LEVEL_ZERO level-zero) + if(LEVEL_ZERO_FOUND) + pkg_get_variable(LEVEL_ZERO_LIBRARY_DIR level-zero libdir) + pkg_get_variable(LEVEL_ZERO_INCLUDE_DIR level-zero includedir) + add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) + set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so") + set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level-zero/ze_api.h") + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level-zero/") + endif(LEVEL_ZERO_FOUND) +else(PKG_CONFIG_FOUND) + +endif(PKG_CONFIG_FOUND) \ No newline at end of file diff --git a/parsec/include/parsec/parsec_options.h.in b/parsec/include/parsec/parsec_options.h.in index efed86686..3bfd8b0cd 100644 --- a/parsec/include/parsec/parsec_options.h.in +++ b/parsec/include/parsec/parsec_options.h.in @@ -121,6 +121,7 @@ #cmakedefine PARSEC_HAVE_HWLOC #cmakedefine PARSEC_HAVE_PAPI #cmakedefine PARSEC_HAVE_CUDA +#cmakedefine PARSEC_HAVE_LEVEL_ZERO #cmakedefine PARSEC_HAVE_OPENCL #cmakedefine PARSEC_HAVE_MPI #cmakedefine PARSEC_HAVE_MPI_20 diff --git a/parsec/mca/device/device.h b/parsec/mca/device/device.h index 86cba401b..eeed22b20 100644 --- a/parsec/mca/device/device.h +++ b/parsec/mca/device/device.h @@ -65,7 +65,8 @@ typedef struct parsec_device_base_component_2_0_0 parsec_device_base_component_t #define PARSEC_DEV_CUDA ((uint8_t)(1 << 2)) #define PARSEC_DEV_INTEL_PHI ((uint8_t)(1 << 3)) #define PARSEC_DEV_OPENCL ((uint8_t)(1 << 4)) -#define PARSEC_DEV_TEMPLATE ((uint8_t)(1 << 5)) +#define PARSEC_DEV_LEVEL_ZERO ((uint8_t)(1 << 5)) +#define PARSEC_DEV_TEMPLATE ((uint8_t)(1 << 6)) #define PARSEC_DEV_ALL ((uint8_t) 0x1f) #define PARSEC_DEV_ANY_TYPE ((uint8_t) 0x1f) #define PARSEC_DEV_MAX_NB_TYPE (6) diff --git a/parsec/mca/device/level_zero/ValidateModule.CMake b/parsec/mca/device/level_zero/ValidateModule.CMake new file mode 100644 index 000000000..88b5c21c0 --- /dev/null +++ b/parsec/mca/device/level_zero/ValidateModule.CMake @@ -0,0 +1,15 @@ +# For now assume that the upper level did the CUDA search and that all +# necessary dependencies on CUDA have been correctly setup + +if( PARSEC_HAVE_LEVEL_ZERO ) + SET(MCA_${COMPONENT}_${MODULE} ON) + FILE(GLOB MCA_${COMPONENT}_${MODULE}_SOURCES ${MCA_BASE_DIR}/${COMPONENT}/${MODULE}/[^\\.]*.c) + SET(MCA_${COMPONENT}_${MODULE}_CONSTRUCTOR "${COMPONENT}_${MODULE}_static_component") + install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero.h + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero_internal.h + DESTINATION include/parsec/mca/device/level_zero ) +else (PARSEC_HAVE_LEVEL_ZERO) + MESSAGE(STATUS "Module ${MODULE} not selectable: does not have LEVEL_ZERO") + SET(MCA_${COMPONENT}_${MODULE} OFF) +endif(PARSEC_HAVE_LEVEL_ZERO) diff --git a/parsec/mca/device/level_zero/device_level_zero.h b/parsec/mca/device/level_zero/device_level_zero.h new file mode 100644 index 000000000..2937b46fb --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#ifndef PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED +#define PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED + +#include "parsec/parsec_internal.h" +#include "parsec/class/parsec_object.h" +#include "parsec/mca/device/device.h" + +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/class/list_item.h" +#include "parsec/class/list.h" +#include "parsec/class/fifo.h" +#include "parsec/mca/device/device_gpu.h" + +#include + +BEGIN_C_DECLS + +struct parsec_level_zero_task_s; +typedef struct parsec_level_zero_task_s parsec_level_zero_task_t; + +struct parsec_level_zero_exec_stream_s; +typedef struct parsec_level_zero_exec_stream_s parsec_level_zero_exec_stream_t; + +struct parsec_device_level_zero_module_s; +typedef struct parsec_device_level_zero_module_s parsec_device_level_zero_module_t; + +struct parsec_level_zero_workspace_s; +typedef struct parsec_level_zero_workspace_s parsec_level_zero_workspace_t; + +extern parsec_device_base_component_t parsec_device_level_zero_component; + +struct parsec_level_zero_task_s { + parsec_gpu_task_t super; +}; + +struct parsec_device_level_zero_module_s { + parsec_device_gpu_module_t super; + uint8_t level_zero_index; + ze_driver_handle_t ze_driver; + ze_device_handle_t ze_device; + ze_context_handle_t ze_context; +}; + +PARSEC_OBJ_CLASS_DECLARATION(parsec_device_level_zero_module_t); + +struct parsec_level_zero_exec_stream_s { + parsec_gpu_exec_stream_t super; + /* There is exactly one task per active event (max_events being the uppoer bound). + * Upon event completion the complete_stage function associated with the task is + * called, and this will decide what is going on next with the task. If the task + * remains in the system the function is supposed to update it. + */ + ze_event_handle_t *events; + ze_event_pool_handle_t ze_event_pool; + ze_command_list_handle_t level_zero_cl; +}; + +/**************************************************** + ** GPU-DATA Specific Starts Here ** + ****************************************************/ +PARSEC_DECLSPEC extern int parsec_level_zero_output_stream; + +/** + * Overload the default data_copy_t with a GPU specialized type + */ +typedef parsec_data_copy_t parsec_gpu_data_copy_t; + +END_C_DECLS + +#define PARSEC_LEVEL_ZERO_CHECK_ERROR(STR, ERROR, CODE) \ + do { \ + if( ZE_RESULT_SUCCESS != (ERROR) ) { \ + parsec_warning( "%s:%d %s returns Error 0x%x", __FILE__, __LINE__,\ + (STR), (ERROR) ); \ + CODE; \ + } \ + } while(0) + +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ + +#endif /* PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED */ diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c new file mode 100644 index 000000000..989ed8214 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -0,0 +1,341 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/parsec_internal.h" +#include "parsec/sys/atomic.h" + +#include "parsec/utils/mca_param.h" +#include "parsec/constants.h" + +#include "parsec/runtime.h" +#include "parsec/data_internal.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#include "parsec/profiling.h" +#include "parsec/execution_stream.h" +#include "parsec/arena.h" +#include "parsec/scheduling.h" +#include "parsec/utils/debug.h" +#include "parsec/utils/argv.h" +#include "parsec/utils/zone_malloc.h" +#include "parsec/class/fifo.h" + +#include + +PARSEC_OBJ_CLASS_INSTANCE(parsec_device_level_zero_module_t, parsec_device_module_t, NULL, NULL); + +static int device_level_zero_component_open(void); +static int device_level_zero_component_close(void); +static int device_level_zero_component_query(mca_base_module_2_0_0_t **module, int *priority); +static int device_level_zero_component_register(void); + +int use_level_zero_index, use_level_zero; +int level_zero_mask, level_zero_nvlink_mask; +int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; + +int32_t parsec_LEVEL_ZERO_sort_pending_list = 0; + +char* level_zero_lib_path = NULL; + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +parsec_device_base_component_t parsec_device_level_zero_component = { + /* First, the mca_component_t struct containing meta information + about the component itself */ + + { + PARSEC_DEVICE_BASE_VERSION_2_0_0, + + /* Component name and version */ + "level_zero", + /* Component options */ +#if defined(PARSEC_HAVE_PEER_DEVICE_MEMORY_ACCESS) + "+peer_access" +#endif + "", + PARSEC_VERSION_MAJOR, + PARSEC_VERSION_MINOR, + + /* Component open and close functions */ + device_level_zero_component_open, + device_level_zero_component_close, + device_level_zero_component_query, + /*< specific query to return the module and add it to the list of available modules */ + device_level_zero_component_register, + "", /*< no reserve */ + }, + { + /* The component has no metadata */ + MCA_BASE_METADATA_PARAM_NONE, + "", /*< no reserve */ + }, + NULL +}; + +mca_base_component_t * device_level_zero_static_component(void) +{ + return (mca_base_component_t *)&parsec_device_level_zero_component; +} + +static int device_level_zero_component_query(mca_base_module_t **module, int *priority) +{ + int i, j, rc; + ze_device_handle_t *devices = NULL; + ze_driver_handle_t *allDrivers = NULL; + + *module = NULL; + *priority = 0; + if( 0 == use_level_zero ) { + return MCA_SUCCESS; + } + parsec_gpu_init_profiling(); + + if( use_level_zero >= 1) { + uint32_t driverCount = 0; + uint32_t totalDeviceCount = 0, maxDeviceCount = 0; + + // Discover all the driver instances + zeDriverGet(&driverCount, NULL); + + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + zeDriverGet(&driverCount, allDrivers); + + for(uint32_t did = 0; did < driverCount; ++did ) { + uint32_t deviceCount = 0; + zeDeviceGet(allDrivers[did], &deviceCount, NULL); + totalDeviceCount += deviceCount; + if(maxDeviceCount < deviceCount) + maxDeviceCount = deviceCount; + } + + use_level_zero = totalDeviceCount < (uint32_t)use_level_zero ? (int)totalDeviceCount : use_level_zero; + + if(use_level_zero > 0) { + parsec_device_level_zero_component.modules = + (parsec_device_module_t **)calloc(use_level_zero + 1, + sizeof(parsec_device_module_t *)); + devices = (ze_device_handle_t *)malloc(maxDeviceCount * sizeof(ze_device_handle_t)); + + i = j = 0; + for(uint32_t did = 0; i < use_level_zero && did < driverCount; ++did ) { + uint32_t deviceCount = maxDeviceCount; + zeDeviceGet(allDrivers[did], &deviceCount, devices); + for(uint32_t devid = 0; i < use_level_zero && devid < deviceCount; devid++) { + ze_device_properties_t device_properties; + zeDeviceGetProperties(devices[devid], &device_properties); + if( ZE_DEVICE_TYPE_GPU != device_properties.type) { continue; } + if( !((1 << i) & level_zero_mask) ) { i++; continue; } + rc = parsec_level_zero_module_init(i, allDrivers[did], devices[devid], &device_properties, + &parsec_device_level_zero_component.modules[j]); + if( PARSEC_SUCCESS != rc ) { + assert( NULL == parsec_device_level_zero_component.modules[j] ); + continue; + } + parsec_device_level_zero_component.modules[j]->component = &parsec_device_level_zero_component; + j++; /* next available spot */ + parsec_device_level_zero_component.modules[j] = NULL; + i++; + } + } + } else + parsec_device_level_zero_component.modules = NULL; + } else + parsec_device_level_zero_component.modules = NULL; + + if(NULL != devices) + free(devices); + if(NULL != allDrivers) + free(allDrivers); + + parsec_gpu_enable_debug(); + + /* module type should be: const mca_base_module_t ** */ + void *ptr = parsec_device_level_zero_component.modules; + *priority = 10; + *module = (mca_base_module_t *)ptr; + + return MCA_SUCCESS; +} + +static int device_level_zero_component_register(void) +{ + use_level_zero_index = parsec_mca_param_reg_int_name("device_level_zero", "enabled", + "The number of LEVEL_ZERO device to enable for the next PaRSEC context (-1 for all available)", + false, false, -1, &use_level_zero); + (void)parsec_mca_param_reg_int_name("device_level_zero", "mask", + "The bitwise mask of LEVEL_ZERO devices to be enabled (default all)", + false, false, 0xffffffff, &level_zero_mask); + (void)parsec_mca_param_reg_int_name("device_level_zero", "nvlink_mask", + "What devices are allowed to use NVLINK if available (default all)", + false, false, 0xffffffff, &level_zero_nvlink_mask); + (void)parsec_mca_param_reg_int_name("device_level_zero", "verbose", + "Set the verbosity level of the LEVEL_ZERO device (negative value: use debug verbosity), higher is less verbose)\n", + false, false, -1, &parsec_gpu_verbosity); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_block_size", + "The LEVEL_ZERO memory page for PaRSEC internal management (in bytes).", + false, false, 512*1024, &level_zero_memory_block_size); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_use", + "The percentage of the total GPU memory to be used by this PaRSEC context", + false, false, 95, &level_zero_memory_percentage); + (void)parsec_mca_param_reg_int_name("device_level_zero", "memory_number_of_blocks", + "Alternative to device_level_zero_memory_use: sets exactly the number of blocks to allocate (-1 means to use a percentage of the available memory)", + false, false, -1, &level_zero_memory_number_of_blocks); + (void)parsec_mca_param_reg_int_name("device_level_zero", "max_number_of_ejected_data", + "Sets up the maximum number of blocks that can be ejected from GPU memory", + false, false, MAX_PARAM_COUNT, &parsec_GPU_d2h_max_flows); + (void)parsec_mca_param_reg_int_name("device_level_zero", "sort_pending_tasks", + "Boolean to let the GPU engine sort the first pending tasks stored in the list", + false, false, 0, &parsec_LEVEL_ZERO_sort_pending_list); +#if defined(PARSEC_PROF_TRACE) + (void)parsec_mca_param_reg_int_name("device_level_zero", "one_profiling_stream_per_level_zero_stream", + "Boolean to separate the profiling of each level_zero stream into a single profiling stream", + false, false, 0, &parsec_device_gpu_one_profiling_stream_per_gpu_stream); +#endif + + /* If LEVEL_ZERO was not requested avoid initializing the devices */ + return (0 == use_level_zero ? MCA_ERROR : MCA_SUCCESS); +} + +/** + * Open LEVEL_ZERO and check that devices are available and ready to be used. This operation should + * only be done once during the initialization, and the devices should from there on be managed + * by PaRSEC. + */ +static int device_level_zero_component_open(void) +{ + ze_result_t ze_rc; + ze_driver_handle_t *allDrivers = NULL; + int ndevices = 0; + uint32_t driverCount = 0; + + if( 0 <= use_level_zero ) { + return MCA_ERROR; /* Nothing to do around here */ + } + + ze_rc = zeInit( 0 ); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeInit ", ze_rc, + { + parsec_mca_param_set_int(use_level_zero_index, 0); + return MCA_ERROR; + } ); + + // Discover all the driver instances + ze_rc = zeDriverGet(&driverCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, + { + parsec_mca_param_set_int(use_level_zero_index, 0); + return MCA_ERROR; + } ); + if(driverCount > 0) { + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + ze_rc = zeDriverGet(&driverCount, allDrivers); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, + { + free(allDrivers); + parsec_mca_param_set_int(use_level_zero_index, 0); + return MCA_ERROR; + } ); + + for( uint32_t did = 0; did < driverCount; ++did ) { + uint32_t deviceCount = 0; + ze_rc = zeDeviceGet(allDrivers[did], &deviceCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet ", ze_rc, + { + free(allDrivers); + parsec_mca_param_set_int(use_level_zero_index, 0); + return MCA_ERROR; + } ); + ndevices += (int)deviceCount; + } + + free(allDrivers); + } + + + if( ndevices > use_level_zero ) { + if( 0 < use_level_zero_index ) { + ndevices = use_level_zero; + } + } else if (ndevices < use_level_zero ) { + if( 0 < use_level_zero_index ) { + parsec_warning("User requested %d LEVEL_ZERO devices, but only %d are available on %s\n." + " PaRSEC will enable all %d of them.", + use_level_zero, ndevices, parsec_hostname, ndevices); + parsec_mca_param_set_int(use_level_zero_index, ndevices); + } + } + + /* Update the number of GPU for the upper layer */ + use_level_zero = ndevices; + if( 0 == ndevices ) { + return -1; + } + + return MCA_SUCCESS; +} + +/** + * Remove all LEVEL_ZERO devices from the PaRSEC available devices, and turn them off. + * At the end of this function all LEVEL_ZERO initialization in the context of PaRSEC + * should be undone, and pending tasks either completed or transferred to another + * chore (if available), and all LEVEL_ZERO resources (events, streams and memory) + * released. + */ +static int device_level_zero_component_close(void) +{ + parsec_device_level_zero_module_t* cdev; + int i, rc; + + if( NULL == parsec_device_level_zero_component.modules ) { /* No devices */ + return PARSEC_SUCCESS; + } + + for( i = 0; NULL != (cdev = (parsec_device_level_zero_module_t*)parsec_device_level_zero_component.modules[i]); i++ ) { + parsec_device_level_zero_component.modules[i] = NULL; + + rc = parsec_level_zero_module_fini((parsec_device_module_t*)cdev); + if( PARSEC_SUCCESS != rc ) { + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] Failed to release resources on LEVEL_ZERO device\n", + cdev->level_zero_index); + } + + /* unregister the device from PaRSEC */ + rc = parsec_mca_device_remove((parsec_device_module_t*)cdev); + if( PARSEC_SUCCESS != rc ) { + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] Failed to unregister LEVEL_ZERO device %d\n", + cdev->level_zero_index, cdev->level_zero_index); + } + + free(cdev); + } + +#if defined(PARSEC_DEBUG_NOISIER) + /* Check that no LEVEL_ZERO devices are still registered with PaRSEC */ + for(i = 0; i < parsec_mca_device_enabled(); i++) { + if( NULL == (cdev = (parsec_device_level_zero_module_t*)parsec_mca_device_get(i)) ) continue; + if(PARSEC_DEV_LEVEL_ZERO != cdev->super.super.type) continue; + + PARSEC_DEBUG_VERBOSE(0, parsec_gpu_output_stream, + "GPU[%d] LEVEL_ZERO device still registered with PaRSEC at the end of LEVEL_ZERO finalize.\n" + " Please contact the developers or fill an issue.\n", + cdev->level_zero_index); + } +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + + if( parsec_device_output != parsec_gpu_output_stream ) + parsec_output_close(parsec_gpu_output_stream); + parsec_gpu_output_stream = parsec_device_output; + + if ( level_zero_lib_path ) { + free(level_zero_lib_path); + } + + return PARSEC_SUCCESS; +} diff --git a/parsec/mca/device/level_zero/device_level_zero_internal.h b/parsec/mca/device/level_zero/device_level_zero_internal.h new file mode 100644 index 000000000..c9c8725f9 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_internal.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#ifndef PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED +#define PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED + +#include "parsec/mca/device/level_zero/device_level_zero.h" + +#if defined(PARSEC_HAVE_LEVEL_ZERO) + +BEGIN_C_DECLS + +#if defined(PARSEC_PROF_TRACE) +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN 0x0001 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT 0x0002 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN 0x0004 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_EXEC 0x0008 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE 0x0010 +#define PARSEC_PROFILE_LEVEL_ZERO_TRACK_PREFETCH 0x0020 + +extern int parsec_level_zero_trackable_events; +extern int parsec_level_zero_movein_key_start; +extern int parsec_level_zero_movein_key_end; +extern int parsec_level_zero_moveout_key_start; +extern int parsec_level_zero_moveout_key_end; +extern int parsec_level_zero_own_GPU_key_start; +extern int parsec_level_zero_own_GPU_key_end; +extern int parsec_level_zero_allocate_memory_key; +extern int parsec_level_zero_free_memory_key; +extern int parsec_level_zero_use_memory_key_start; +extern int parsec_level_zero_use_memory_key_end; +extern int parsec_level_zero_prefetch_key_start; +extern int parsec_level_zero_prefetch_key_end; +extern int parsec_device_level_zero_one_profiling_stream_per_level_zero_stream; +#endif /* defined(PROFILING) */ + +#define GPU_TASK_TYPE_KERNEL 0x0000 +#define GPU_TASK_TYPE_D2HTRANSFER 0x1000 +#define GPU_TASK_TYPE_PREFETCH 0x2000 +#define GPU_TASK_TYPE_WARMUP 0x4000 +#define GPU_TASK_TYPE_D2D_COMPLETE 0x8000 + +/* From MCA parameters */ +extern int use_level_zero_index, use_level_zero; +extern int level_zero_mask, level_zero_verbosity; +extern int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; +extern char* level_zero_lib_path; +extern int32_t parsec_LEVEL_ZERO_d2h_max_flows; +extern int32_t parsec_LEVEL_ZERO_sort_pending_list; + +PARSEC_DECLSPEC extern const parsec_device_module_t parsec_device_level_zero_module; + +/**************************************************** + ** GPU-DATA Specific Starts Here ** + ****************************************************/ + +int parsec_level_zero_module_init( int device_id, ze_driver_handle_t ze_driver, ze_device_handle_t ze_device, + ze_device_properties_t *device_properties, parsec_device_module_t** module ); +int parsec_level_zero_module_fini(parsec_device_module_t* device); + +END_C_DECLS + +#if defined(PARSEC_PROF_TRACE) +typedef struct { + uint64_t size; + uint64_t data_key; + uint64_t dc_id; +} parsec_device_level_zero_memory_prof_info_t; +#define PARSEC_DEVICE_LEVEL_ZERO_MEMORY_PROF_INFO_CONVERTER "size{int64_t};data_key{uint64_t};dc_id{uint64_t}" +#endif /* PARSEC_PROF_TRACE */ + +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ + +#endif /* PARSEC_DEVICE_LEVEL_ZERO_INTERNAL_H_HAS_BEEN_INCLUDED */ diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c new file mode 100644 index 000000000..bbb0c3771 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -0,0 +1,2667 @@ +/* + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/parsec_internal.h" +#include "parsec/sys/atomic.h" + +#include "parsec/utils/mca_param.h" +#include "parsec/constants.h" + +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/runtime.h" +#include "parsec/data_internal.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#include "parsec/profiling.h" +#include "parsec/execution_stream.h" +#include "parsec/arena.h" +#include "parsec/scheduling.h" +#include "parsec/utils/debug.h" +#include "parsec/utils/argv.h" +#include "parsec/utils/zone_malloc.h" +#include "parsec/class/fifo.h" + +#include + +static int parsec_level_zero_data_advise(parsec_device_module_t *dev, parsec_data_t *data, int advice); +static int +parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* gpu_device, + int memory_percentage, + int number_of_elements, + size_t eltsize ); +static int parsec_level_zero_memory_release( parsec_device_level_zero_module_t* gpu_device ); +static int parsec_level_zero_flush_lru( parsec_device_module_t *device ); + +static void* level_zero_find_incarnation(parsec_device_level_zero_module_t* level_zero_device, + const char* fname) +{ + char library_name[FILENAME_MAX], *env; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + void *fn = NULL; + char** argv = NULL; + + /** + * Prepare the list of PATH or FILE to be searched for a LEVEL_ZERO shared library. + * In any case this list might be a list of ; separated possible targets, + * where each target can be either a directory or a specific file. + */ + env = getenv("PARSEC_CUCORES_LIB"); + if( NULL != env ) { + argv = parsec_argv_split(env, ';'); + } else if( NULL != level_zero_lib_path ) { + argv = parsec_argv_split(level_zero_lib_path, ';'); + } + + fn = parsec_device_find_function(fname, library_name, (const char**)argv); + if( NULL == fn ) { /* look for the function with lesser capabilities */ + parsec_warning("No function '%s' found for LEVEL_ZERO device %s", fname, gpu_device->super.name); + } + + if( NULL != argv ) + parsec_argv_free(argv); + + return fn; +} + +/** + * Register a taskpool with a device by checking that the device + * supports the dynamic function required by the different incarnations. + * If multiple devices of the same type exists we assume thay all have + * the same capabilities. + */ +static int +parsec_level_zero_taskpool_register(parsec_device_module_t* device, + parsec_taskpool_t* tp) +{ + parsec_device_level_zero_module_t* gpu_device = (parsec_device_level_zero_module_t*)device; + int32_t rc = PARSEC_ERR_NOT_FOUND; + uint32_t i, j; + + /** + * Detect if a particular chore has a dynamic load dependency and if yes + * load the corresponding module and find the function. + */ + assert(PARSEC_DEV_LEVEL_ZERO == device->type); + assert(tp->devices_index_mask & (1 << device->device_index)); + + for( i = 0; i < tp->nb_task_classes; i++ ) { + const parsec_task_class_t* tc = tp->task_classes_array[i]; + __parsec_chore_t* chores = (__parsec_chore_t*)tc->incarnations; + for( j = 0; NULL != chores[j].hook; j++ ) { + if( chores[j].type != device->type ) + continue; + if( NULL != chores[j].dyld_fn ) { + /* the function has been set for another device of the same type */ + return PARSEC_SUCCESS; + } + if ( NULL == chores[j].dyld ) { + chores[j].dyld_fn = NULL; /* No dynamic support required for this kernel */ + rc = PARSEC_SUCCESS; + } else { + void* devf = level_zero_find_incarnation(gpu_device, chores[j].dyld); + if( NULL != devf ) { + chores[j].dyld_fn = devf; + rc = PARSEC_SUCCESS; + } + } + } + } + if( PARSEC_SUCCESS != rc ) { + tp->devices_index_mask &= ~(1 << device->device_index); /* drop support for this device */ + parsec_debug_verbose(10, parsec_gpu_output_stream, + "Device %d (%s) disabled for taskpool %p", device->device_index, device->name, tp); + } + return rc; +} + +static int +parsec_level_zero_taskpool_unregister(parsec_device_module_t* device, parsec_taskpool_t* tp) +{ + (void)device; (void)tp; + return PARSEC_SUCCESS; +} + +/** + * Attach a device to a PaRSEC context. A device can only be attached to + * a single context at the time. + */ +static int +parsec_device_level_zero_attach( parsec_device_module_t* device, + parsec_context_t* context ) +{ + return parsec_mca_device_add(context, device); +} + +/** + * Detach a device from a context. Both the context and the device remain + * valid, they are simply disconnected. + * This function should only be called once all tasks and all data related to the + * context has been removed from the device. + */ +static int +parsec_device_level_zero_detach( parsec_device_module_t* device, + parsec_context_t* context ) +{ + (void)context; + return parsec_mca_device_remove(device); +} + +int parsec_level_zero_module_init( int dev_id, ze_driver_handle_t ze_driver, ze_device_handle_t ze_device, + ze_device_properties_t *prop, parsec_device_module_t** module ) +{ + int streaming_multiprocessor, len; + parsec_device_level_zero_module_t* level_zero_device; + parsec_device_gpu_module_t* gpu_device; + parsec_device_module_t* device; + ze_result_t ze_rc; + int show_caps_index, show_caps = 0, j, k; + char *szName; + float clockRate; + + show_caps_index = parsec_mca_param_find("device", NULL, "show_capabilities"); + if(0 < show_caps_index) { + parsec_mca_param_lookup_int(show_caps_index, &show_caps); + } + + *module = NULL; + + szName = prop->name; + clockRate = prop->coreClockRate/1e3f; + streaming_multiprocessor = prop->numThreadsPerEU; + + level_zero_device = (parsec_device_level_zero_module_t*)calloc(1, sizeof(parsec_device_level_zero_module_t)); + gpu_device = &level_zero_device->super; + device = &gpu_device->super; + PARSEC_OBJ_CONSTRUCT(level_zero_device, parsec_device_level_zero_module_t); + level_zero_device->level_zero_index = (uint8_t)dev_id; + level_zero_device->ze_driver = ze_driver; + level_zero_device->ze_device = ze_device; + + // Create context + ze_context_desc_t ctxtDesc = { + ZE_STRUCTURE_TYPE_CONTEXT_DESC, + NULL, + 0 + }; + + ze_rc = zeContextCreate(ze_driver, &ctxtDesc, &level_zero_device->ze_context); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeContextCreate ", ze_rc, + { + free(level_zero_device); + return PARSEC_ERROR; + } ); + + len = asprintf(&gpu_device->super.name, "%s ZE(%d)", szName, dev_id); + if(-1 == len) + gpu_device->super.name = ""; + gpu_device->data_avail_epoch = 0; + + gpu_device->max_exec_streams = PARSEC_MAX_STREAMS; + gpu_device->exec_stream = + (parsec_gpu_exec_stream_t**)malloc(gpu_device->max_exec_streams * sizeof(parsec_gpu_exec_stream_t*)); + gpu_device->exec_stream[0] = + (parsec_gpu_exec_stream_t*)malloc( gpu_device->max_exec_streams * sizeof(parsec_level_zero_exec_stream_t)); + for(j = 1; j < gpu_device->max_exec_streams; j++) + gpu_device->exec_stream[j] = (parsec_gpu_exec_stream_t*)( + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[0]+j); + + // Discover all command queue groups + uint32_t cmdqueueGroupCount = 0; + zeDeviceGetCommandQueueGroupProperties(level_zero_device->ze_device, &cmdqueueGroupCount, NULL); + + ze_command_queue_group_properties_t* cmdqueueGroupProperties = (ze_command_queue_group_properties_t*) + malloc(cmdqueueGroupCount * sizeof(ze_command_queue_group_properties_t)); + zeDeviceGetCommandQueueGroupProperties(level_zero_device->ze_device, &cmdqueueGroupCount, cmdqueueGroupProperties); + // Find a command queue type that support compute + uint32_t computeQueueGroupOrdinal = cmdqueueGroupCount; + uint32_t copyQueueGroupOrdinal = cmdqueueGroupCount; + for( uint32_t i = 0; + i < cmdqueueGroupCount && + (computeQueueGroupOrdinal == cmdqueueGroupCount || + copyQueueGroupOrdinal == cmdqueueGroupCount); + ++i ) { + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE ) { + computeQueueGroupOrdinal = i; + } + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY ) { + copyQueueGroupOrdinal = i; + } + } + //TODO: it might be more in line with the design to create different command queues for copy + // and compute than using the existing queues. + if( computeQueueGroupOrdinal == cmdqueueGroupCount ) { + parsec_warning( "level zero device: unable to find a Queue Group with COMPUTE flag"); + goto release_device; + } + if( copyQueueGroupOrdinal == cmdqueueGroupCount ) { + parsec_warning( "level zero device: unable to find a Queue Group with COMPUTE flag"); + goto release_device; + } + + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t* level_zero_stream = + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + ze_command_queue_desc_t commandQueueDesc = { + ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, + NULL, + (uint32_t)-1, + 0, // index + 0, // flags + ZE_COMMAND_QUEUE_MODE_DEFAULT, + ZE_COMMAND_QUEUE_PRIORITY_NORMAL + }; + if( j < 2 ) + commandQueueDesc.ordinal = copyQueueGroupOrdinal; + else + commandQueueDesc.ordinal = computeQueueGroupOrdinal; + ze_rc = zeCommandListCreateImmediate(level_zero_device->ze_context, level_zero_device->ze_device, + &commandQueueDesc, + &level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreateImmediate ", ze_rc, + {goto release_device;} ); + exec_stream->workspace = NULL; + PARSEC_OBJ_CONSTRUCT(&exec_stream->infos, parsec_info_object_array_t); + parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream); + exec_stream->max_events = PARSEC_MAX_EVENTS_PER_STREAM; + exec_stream->executed = 0; + exec_stream->start = 0; + exec_stream->end = 0; + exec_stream->name = NULL; + exec_stream->fifo_pending = (parsec_list_t*)PARSEC_OBJ_NEW(parsec_list_t); + PARSEC_OBJ_CONSTRUCT(exec_stream->fifo_pending, parsec_list_t); + exec_stream->tasks = (parsec_gpu_task_t**)malloc(exec_stream->max_events + * sizeof(parsec_gpu_task_t*)); + level_zero_stream->events = (ze_event_handle_t*)malloc(exec_stream->max_events * sizeof(ze_event_handle_t)); + + // Create event pool + ze_event_pool_desc_t eventPoolDesc = { + ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, + NULL, + ZE_EVENT_POOL_FLAG_HOST_VISIBLE, // all events in pool are visible to Host + 1 // count + }; + zeEventPoolCreate(level_zero_device->ze_context, &eventPoolDesc, 0, NULL, + &level_zero_stream->ze_event_pool); + + /* and the corresponding events */ + for( k = 0; k < exec_stream->max_events; k++ ) { + ze_event_desc_t eventDesc = { + ZE_STRUCTURE_TYPE_EVENT_DESC, + NULL, + 0, // index + 0, // no additional memory/cache coherency required on signal + ZE_EVENT_SCOPE_FLAG_HOST // ensure memory coherency across device and Host after event completes + }; + level_zero_stream->events[k] = NULL; + exec_stream->tasks[k] = NULL; + ze_rc = zeEventCreate(level_zero_stream->ze_event_pool, &eventDesc, &(level_zero_stream->events[k])); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventCreate ", ze_rc, + {goto release_device;} ); + } + if(j == 0) { + len = asprintf(&exec_stream->name, "h2d(%d)", j); + if(-1 == len) + exec_stream->name = "h2d"; + } else if(j == 1) { + len = asprintf(&exec_stream->name, "d2h(%d)", j); + if(-1 == len) + exec_stream->name = "d2h"; + } else { + len = asprintf(&exec_stream->name, "level_zero(%d)", j); + if(-1 == len) + exec_stream->name = "level_zero"; + } +#if defined(PARSEC_PROF_TRACE) + /* Each 'exec' stream gets its own profiling stream, except IN and OUT stream that share it. + * It's good to separate the exec streams to know what was submitted to what stream + * We don't have this issue for the IN and OUT streams because types of event discriminate + * what happens where, and separating them consumes memory and increases the number of + * events that needs to be matched between streams because we cannot differentiate some + * ends between IN or OUT, so they are all logged on the same stream. */ + if(j == 0 || (parsec_device_gpu_one_profiling_stream_per_gpu_stream == 1 && j != 1)) + exec_stream->profiling = parsec_profiling_stream_init( 2*1024*1024, PARSEC_PROFILE_STREAM_STR, dev_id, j ); + else + exec_stream->profiling = gpu_device->exec_stream[0]->profiling; + if(j == 0) { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } else if(j == 1) { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } else { + exec_stream->prof_event_track_enable = parsec_gpu_trackable_events & ( PARSEC_PROFILE_LEVEL_ZERO_TRACK_EXEC | PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE ); + } +#endif /* defined(PARSEC_PROF_TRACE) */ + } + + device->type = PARSEC_DEV_LEVEL_ZERO; + device->executed_tasks = 0; + device->transferred_data_in = 0; + device->d2d_transfer = 0; + device->transferred_data_out = 0; + device->required_data_in = 0; + device->required_data_out = 0; + + device->attach = parsec_device_level_zero_attach; + device->detach = parsec_device_level_zero_detach; + device->memory_register = NULL; // TODO: could not find a memory register equivalent in level zero + device->memory_unregister = NULL; // TODO: could not find a memory unregister equivalent in level zero + device->taskpool_register = parsec_level_zero_taskpool_register; + device->taskpool_unregister = parsec_level_zero_taskpool_unregister; + device->data_advise = parsec_level_zero_data_advise; + device->memory_release = parsec_level_zero_flush_lru; + + device->device_hweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_tweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_sweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + device->device_dweight = (float)streaming_multiprocessor * (float)clockRate * 2e-3f; + + /* Initialize internal lists */ + PARSEC_OBJ_CONSTRUCT(&gpu_device->gpu_mem_lru, parsec_list_t); + PARSEC_OBJ_CONSTRUCT(&gpu_device->gpu_mem_owned_lru, parsec_list_t); + PARSEC_OBJ_CONSTRUCT(&gpu_device->pending, parsec_fifo_t); + + gpu_device->sort_starting_p = NULL; + gpu_device->peer_access_mask = 0; /* No GPU to GPU direct transfer by default */ + + if( PARSEC_SUCCESS != parsec_level_zero_memory_reserve(level_zero_device, + level_zero_memory_percentage, + level_zero_memory_number_of_blocks, + level_zero_memory_block_size) ) { + goto release_device; + } + + if( show_caps ) { + parsec_inform("LEVEL ZERO GPU Device %d: %s\n" + "\tLocation (PCI Bus/Device/Domain): %x:%x.%x\n" + "\tnumThreadsPerEU : %u\n" + "\tphysicalEUSimdWidth : %u\n" + "\tnumEUsPerSubslice : %u\n" + "\tnumSubslicesPerSlice: %u\n" + "\tnumSlices: %u\n" + "\tcoreClockRate (GHz): %2.2f\n" + "\tpeak Gflops : double %2.3f, single %2.3f tensor %2.3f half %2.3f\n", + level_zero_device->level_zero_index, device->name, + prop->deviceId, prop->subdeviceId, prop->vendorId, + streaming_multiprocessor, + prop->physicalEUSimdWidth, + prop->numEUsPerSubslice, + prop->numSubslicesPerSlice, + prop->numSlices, + clockRate*1e-3, + device->device_dweight, device->device_sweight, device->device_tweight, device->device_hweight); + } + + *module = device; + return PARSEC_SUCCESS; + + release_device: + if( NULL != gpu_device->exec_stream) { + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t *level_zero_stream = + (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + + if( NULL != exec_stream->fifo_pending ) { + PARSEC_OBJ_RELEASE(exec_stream->fifo_pending); + } + if( NULL != exec_stream->tasks ) { + free(exec_stream->tasks); exec_stream->tasks = NULL; + } + if( NULL != level_zero_stream->events ) { + for( k = 0; k < exec_stream->max_events; k++ ) { + if( NULL != level_zero_stream->events[k] ) { + (void)zeEventDestroy(level_zero_stream->events[k]); + } + } + free(level_zero_stream->events); level_zero_stream->events = NULL; + zeEventPoolDestroy(level_zero_stream->ze_event_pool); + } + if( NULL != exec_stream->name ) { + free(exec_stream->name); exec_stream->name = NULL; + } +#if defined(PARSEC_PROF_TRACE) + if( NULL != exec_stream->profiling ) { + /* No function to clean the profiling stream. If one is introduced + * some day, remember that exec streams 0 and 1 always share the same + * ->profiling stream, and that all of them share the same + * ->profiling stream if parsec_device_gpu_one_profiling_stream_per_gpu_stream == 0 */ + } +#endif /* defined(PARSEC_PROF_TRACE) */ + } + // All exec_stream_t are allocated in a single malloc in gpu_device->exec_stream[0] + free(gpu_device->exec_stream[0]); + free(gpu_device->exec_stream); + gpu_device->exec_stream = NULL; + } + free(gpu_device); + return PARSEC_ERROR; +} + +int +parsec_level_zero_module_fini(parsec_device_module_t* device) +{ + parsec_device_gpu_module_t* gpu_device = (parsec_device_gpu_module_t*)device; + parsec_device_level_zero_module_t* level_zero_device = (parsec_device_level_zero_module_t*)device; + ze_result_t status; + int j, k; + + /* Release the registered memory */ + parsec_level_zero_memory_release(level_zero_device); + + /* Release pending queue */ + PARSEC_OBJ_DESTRUCT(&gpu_device->pending); + + /* Release all streams */ + for( j = 0; j < gpu_device->max_exec_streams; j++ ) { + parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)gpu_device->exec_stream[j]; + parsec_gpu_exec_stream_t* exec_stream = &level_zero_stream->super; + + exec_stream->executed = 0; + exec_stream->start = 0; + exec_stream->end = 0; + + for( k = 0; k < exec_stream->max_events; k++ ) { + assert( NULL == exec_stream->tasks[k] ); + status = zeEventDestroy(level_zero_stream->events[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(parsec_level_zero_device_fini) level_zeroEventDestroy ", status, + {continue;} ); + } + zeEventPoolDestroy(level_zero_stream->ze_event_pool); + exec_stream->max_events = 0; + free(level_zero_stream->events); level_zero_stream->events = NULL; + free(exec_stream->tasks); exec_stream->tasks = NULL; + free(exec_stream->fifo_pending); exec_stream->fifo_pending = NULL; + /* Release the stream */ + zeCommandListDestroy( level_zero_stream->level_zero_cl ); + free(exec_stream->name); + + /* Release Info object array */ + PARSEC_OBJ_DESTRUCT(&exec_stream->infos); + } + // All exec_stream_t are allocated in a single malloc in gpu_device->exec_stream[0] + free(gpu_device->exec_stream[0]); + free(gpu_device->exec_stream); + gpu_device->exec_stream = NULL; + level_zero_device->level_zero_index = -1; + + /* Cleanup the GPU memory. */ + PARSEC_OBJ_DESTRUCT(&gpu_device->gpu_mem_lru); + PARSEC_OBJ_DESTRUCT(&gpu_device->gpu_mem_owned_lru); + + return PARSEC_SUCCESS; +} + +/** + * This function reserve the memory_percentage of the total device memory for PaRSEC. + * This memory will be managed in chuncks of size eltsize. However, multiple chuncks + * can be reserved in a single allocation. + */ +static int +parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* level_zero_device, + int memory_percentage, + int number_blocks, + size_t eltsize ) +{ + ze_result_t status; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + (void)eltsize; + ze_device_properties_t devProperties; + ze_device_memory_properties_t *devMemProperties; + ze_device_memory_access_properties_t memAccessProperties; + uint32_t count = 0; + + size_t how_much_we_allocate; + size_t initial_free_mem, alignment = 1 << 3; + int memIndex = -1; + uint32_t mem_elem_per_gpu = 0; + + status = zeDeviceGetMemoryAccessProperties(level_zero_device->ze_device, &memAccessProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryAccessProperties ", status, { + return PARSEC_ERROR; + }); + if( 0 == (ZE_MEMORY_ACCESS_CAP_FLAG_RW & memAccessProperties.deviceAllocCapabilities) ) { + parsec_warning("%s:%d -- Device %s does not have memory allocation capabilities with RW access\n", + __FILE__, __LINE__, gpu_device->super.name); + return PARSEC_ERROR; + } + status = zeDeviceGetProperties(level_zero_device->ze_device, &devProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetProperties ", status, { + return PARSEC_ERROR; + }); + status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { + return PARSEC_ERROR; + }); + devMemProperties = (ze_device_memory_properties_t*)malloc(count * sizeof(ze_device_memory_properties_t)); + status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, devMemProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { + free(devMemProperties); + return PARSEC_ERROR; + }); + for(int i = 0; i < (int)count; i++) { + // TODO: better approach would be to keep a list of pointers? + // for now we just take the memory that has the highest amount of memory available + if( memIndex == -1 || devMemProperties[memIndex].totalSize < devMemProperties[i].totalSize) + memIndex = i; + } + initial_free_mem = devProperties.maxMemAllocSize < devMemProperties[memIndex].totalSize ? + devProperties.maxMemAllocSize : devMemProperties[memIndex].totalSize; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "level-zero device %s: initial_free_mem is %lu (device max alloc size is %lu, device memory " + "property total size for memory bank %d is %lu)", gpu_device->super.name, initial_free_mem, + devProperties.maxMemAllocSize, memIndex, devMemProperties[memIndex].totalSize); + free(devMemProperties); devMemProperties = NULL; + + if( number_blocks != -1 ) { + if( number_blocks == 0 ) { + parsec_warning("LEVEL_ZERO[%d] Invalid argument: requesting 0 bytes of memory on LEVEL_ZERO device %s", + gpu_device->super.name, gpu_device->super.name); + return PARSEC_ERROR; + } else { + how_much_we_allocate = number_blocks * eltsize; + } + } else { + /** number_blocks == -1 means memory_percentage is used */ + how_much_we_allocate = (memory_percentage * initial_free_mem) / 100; + } + if( how_much_we_allocate > initial_free_mem ) { + /** Handle the case of jokers who require more than 100% of memory, + * and eleventh case of computer scientists who don't know how + * to divide a number by another + */ + parsec_warning("LEVEL_ZERO[%d] Requested %zd bytes on LEVEL_ZERO device %s, but only %zd bytes are available -- reducing allocation to max available", + level_zero_device->level_zero_index, how_much_we_allocate, gpu_device->super.name, initial_free_mem); + how_much_we_allocate = initial_free_mem; + } + if( how_much_we_allocate < eltsize ) { + /** Handle another kind of jokers entirely, and cases of + * not enough memory on the device + */ + parsec_warning("LEVEL_ZERO[%d] Cannot allocate at least one element on LEVEL_ZERO device %s", + level_zero_device->level_zero_index, gpu_device->super.name); + return PARSEC_ERROR; + } + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = memIndex + }; + +#if defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + size_t free_mem = initial_free_mem; + /* + * We allocate a bunch of tiles that will be used + * during the computations + */ + while( (free_mem > eltsize ) + && ((total_mem - free_mem) < how_much_we_allocate) ) { + parsec_gpu_data_copy_t* gpu_elem; + void *device_ptr; + + status = zeMemAllocDevice(level_zero_device->ze_context, &memAllocDesc, eltsize, alignment, + level_zero_device->ze_device, &device_ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { break; } ); + gpu_elem = PARSEC_OBJ_NEW(parsec_data_copy_t); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate LEVEL_ZERO copy %p [ref_count %d] for data [%p]", + gpu_device->super.name,gpu_elem, gpu_elem->super.obj_reference_count, NULL); + gpu_elem->device_private = (void*)(long)device_ptr; + gpu_elem->flags |= PARSEC_DATA_FLAG_PARSEC_OWNED; + gpu_elem->device_index = gpu_device->super.device_index; + mem_elem_per_gpu++; + PARSEC_OBJ_RETAIN(gpu_elem); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Retain and insert LEVEL_ZERO copy %p [ref_count %d] in LRU", + gpu_device->super.name, gpu_elem, gpu_elem->super.obj_reference_count); + parsec_list_push_back( &gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_elem ); + free_mem -= alignment * ((size + alignment - 1) / alignment); + } + if( 0 == mem_elem_per_gpu && parsec_list_is_empty( &gpu_device->gpu_mem_lru ) ) { + parsec_warning("GPU[%s] Cannot allocate memory on GPU %s. Skip it!", gpu_device->super.name, gpu_device->super.name); + } + else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u tiles on the GPU memory", + gpu_device->super.name, mem_elem_per_gpu ); + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u tiles on the GPU memory", gpu_device->super.name, mem_elem_per_gpu); +#else + if( NULL == gpu_device->memory ) { + void* base_ptr; + /* We allocate all the memory on the GPU and we use our memory management. */ + /* This computation leads to allocating more than available if we asked for more than GPU memory */ + mem_elem_per_gpu = (how_much_we_allocate + eltsize - 1 ) / eltsize; + size_t total_size = (size_t)mem_elem_per_gpu * eltsize; + + if (total_size > initial_free_mem) { + /* Mapping more than 100% of GPU memory is obviously wrong */ + /* Mapping exactly 100% of the GPU memory ends up producing errors about __global__ function call is not configured */ + /* Mapping 95% works with low-end GPUs like 1060, how much to let available for level_zero runtime, I don't know how to calculate */ + total_size = (size_t)((int)(.9*initial_free_mem / eltsize)) * eltsize; + mem_elem_per_gpu = total_size / eltsize; + } + status = zeMemAllocDevice(level_zero_device->ze_context, &memAllocDesc, total_size, alignment, + level_zero_device->ze_device, &base_ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, + ({ parsec_warning("GPU[%s] Allocating %zu bytes of memory on the GPU device failed", + gpu_device->super.name, total_size); }) ); + + gpu_device->memory = zone_malloc_init( base_ptr, mem_elem_per_gpu, eltsize ); + + if( gpu_device->memory == NULL ) { + parsec_warning("GPU[%s] Cannot allocate memory on GPU %s. Skip it!", + gpu_device->super.name, gpu_device->super.name); + return PARSEC_ERROR; + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s] Allocate %u segments of size %d on the GPU memory", + gpu_device->super.name, mem_elem_per_gpu, eltsize ); + } +#endif + gpu_device->mem_block_size = eltsize; + gpu_device->mem_nb_blocks = mem_elem_per_gpu; + + return PARSEC_SUCCESS; +} + +static void parsec_level_zero_memory_release_list(parsec_device_level_zero_module_t* level_zero_device, + parsec_list_t* list) +{ + parsec_list_item_t* item; +#if defined(PARSEC_DEBUG_VERBOSE) + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; +#endif + + while(NULL != (item = parsec_list_pop_front(list)) ) { + parsec_gpu_data_copy_t* gpu_copy = (parsec_gpu_data_copy_t*)item; + parsec_data_t* original = gpu_copy->original; + + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Release LEVEL_ZERO copy %p (device_ptr %p) [ref_count %d: must be 1], attached to %p, in map %p", + gpu_device->super.name, gpu_copy, gpu_copy->device_private, gpu_copy->super.super + .obj_reference_count, + original, (NULL != original ? original->dc : NULL)); + assert( gpu_copy->device_index == level_zero_device->super.super.device_index ); + + if( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ) { + parsec_warning("GPU[%s] still OWNS the master memory copy for data %d and it is discarding it!", + gpu_device->super.name, original->key); + } + assert(0 != (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + +#if defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + level_zeroFree( gpu_copy->device_private ); +#else + +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_free_memory_key, (int64_t)gpu_copy->device_private, + gpu_device->super.device_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_use_memory_key_end, + (uint64_t)gpu_copy->device_private, + gpu_device->super.device_index, NULL, 0); + } +#endif + zone_free( level_zero_device->super.memory, (void*)gpu_copy->device_private ); +#endif + gpu_copy->device_private = NULL; + + /* At this point the data copies should have no attachement to a data_t. Thus, + * before we get here (aka below parsec_fini), the destructor of the data + * collection must have been called, releasing all the copies. + */ + PARSEC_OBJ_RELEASE(gpu_copy); assert(NULL == gpu_copy); + } +} + +/** + * This function only flushes the data copies pending in LRU, and checks + * (in debug mode) that the entire allocated memory is free to use */ +static int +parsec_level_zero_flush_lru( parsec_device_module_t *device ) +{ + size_t in_use; + parsec_device_gpu_module_t *gpu_device = (parsec_device_gpu_module_t*)device; + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)device; + /* Free all memory on GPU */ + parsec_level_zero_memory_release_list(level_zero_device, &gpu_device->gpu_mem_lru); + parsec_level_zero_memory_release_list(level_zero_device, &gpu_device->gpu_mem_owned_lru); +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) && !defined(_NDEBUG) + if( (in_use = zone_in_use(gpu_device->memory)) != 0 ) { + parsec_warning("GPU[%s] memory leak detected: %lu bytes still allocated on GPU", + device->name, in_use); + assert(0); + } +#endif + return PARSEC_SUCCESS; +} + +/** + * This function release the LEVEL_ZERO memory reserved for this device. + * + * One has to notice that all the data available on the GPU is stored in one of + * the two used to keep track of the allocated data, either the gpu_mem_lru or + * the gpu_mem_owner_lru. Thus, going over all the elements in these two lists + * should be enough to enforce a clean release. + */ +static int +parsec_level_zero_memory_release( parsec_device_level_zero_module_t* level_zero_device ) +{ + ze_result_t status; + + parsec_level_zero_flush_lru(&level_zero_device->super.super); + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + assert( NULL != level_zero_device->super.memory ); + void* ptr = zone_malloc_fini(&level_zero_device->super.memory); + status = zeMemFree(level_zero_device->ze_context, ptr); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, + { parsec_warning("Failed to free the GPU backend memory."); } ); +#endif + + return PARSEC_SUCCESS; +} + +/** + * Try to find memory space to move all data on the GPU. We attach a device_elem to + * a memory_elem as soon as a device_elem is available. If we fail to find enough + * available elements, we push all the elements handled during this allocation + * back into the pool of available device_elem, to be picked up by another call + * (this call will remove them from the current task). + * Returns: + * PARSEC_HOOK_RETURN_DONE: All gpu_mem/mem_elem have been initialized + * PARSEC_HOOK_RETURN_AGAIN: At least one flow is marked under transfer, task cannot be scheduled yet + * PARSEC_HOOK_RETURN_NEXT: The task needs to rescheduled + */ +static inline int +parsec_gpu_data_reserve_device_space( parsec_device_level_zero_module_t* level_zero_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t* temp_loc[MAX_PARAM_COUNT], *gpu_elem, *lru_gpu_elem; + parsec_data_t* master, *oldmaster; + const parsec_flow_t *flow; + int i, j, data_avail_epoch = 0; + parsec_gpu_data_copy_t *gpu_mem_lru_cycling; + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + +#if defined(PARSEC_DEBUG_NOISIER) + char task_name[MAX_TASK_STRLEN]; + parsec_task_snprintf(task_name, MAX_TASK_STRLEN, this_task); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + + /** + * Parse all the input and output flows of data and ensure all have + * corresponding data on the GPU available. + */ + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + gpu_mem_lru_cycling = NULL; + flow = gpu_task->flow[i]; + assert( flow && (flow->flow_index == i) ); + + /* Skip CTL flows only */ + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Investigating flow %s:%d", + gpu_device->super.name, task_name, flow->name, i); + temp_loc[i] = NULL; + if (this_task->data[i].data_in == NULL) + continue; + + master = this_task->data[i].data_in->original; + parsec_atomic_lock(&master->lock); + gpu_elem = PARSEC_DATA_GET_COPY(master, gpu_device->super.device_index); + this_task->data[i].data_out = gpu_elem; + + /* There is already a copy on the device */ + if( NULL != gpu_elem ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Flow %s:%i has a copy on the device %p%s", + gpu_device->super.name, task_name, + flow->name, i, gpu_elem, + gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ? " [in transfer]" : ""); + if ( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Copy %p [ref_count %d] is still in transfer, descheduling...", + gpu_device->super.name, task_name, + gpu_elem, gpu_elem->super.super.obj_reference_count); + SET_HIGHEST_PRIORITY(gpu_task->ec, parsec_execution_context_priority_comparator); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_AGAIN; + } + parsec_atomic_unlock(&master->lock); + continue; + } + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + gpu_elem = PARSEC_OBJ_NEW(parsec_data_copy_t); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Allocate LEVEL_ZERO copy %p sz %d[ref_count %d] for data %p", + gpu_device->super.name, task_name, + gpu_elem, gpu_task->flow_nb_elts[i], gpu_elem->super.super.obj_reference_count, master); + gpu_elem->flags = PARSEC_DATA_FLAG_PARSEC_OWNED | PARSEC_DATA_FLAG_PARSEC_MANAGED; + malloc_data: + assert(0 != (gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + gpu_elem->device_private = zone_malloc(gpu_device->memory, gpu_task->flow_nb_elts[i]); + if( NULL == gpu_elem->device_private ) { +#endif + + find_another_data: + /* Look for a data_copy to free */ + lru_gpu_elem = (parsec_gpu_data_copy_t*)parsec_list_pop_front(&gpu_device->gpu_mem_lru); + if( NULL == lru_gpu_elem ) { + /* We can't find enough room on the GPU. Insert the tiles in the begining of + * the LRU (in order to be reused asap) and return without scheduling the task. + */ +#if defined(PARSEC_DEBUG_NOISIER) + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]:%s:\tRequest space on GPU failed for flow %s index %d/%d for task %s", + gpu_device->super.name, task_name, + flow->name, i, this_task->task_class->nb_flows, task_name ); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + for( j = 0; j < i; j++ ) { + /* This flow could be a control flow */ + if( NULL == temp_loc[j] ) continue; + /* This flow could be non-parsec-owned, in which case we can't reclaim it */ + if( 0 == (temp_loc[j]->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\tAdd copy %p [ref_count %d] back to the LRU list", + gpu_device->super.name, task_name, + temp_loc[j], temp_loc[j]->super.super.obj_reference_count); + /* push them at the head to reach them again at the next iteration */ + parsec_list_push_front(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)temp_loc[j]); + } +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + PARSEC_OBJ_RELEASE(gpu_elem); +#endif + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } + + PARSEC_LIST_ITEM_SINGLETON(lru_gpu_elem); + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Evaluate LRU-retrieved LEVEL_ZERO copy %p [ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count, + lru_gpu_elem->original); + + /* If there are pending readers, let the gpu_elem loose. This is a weak coordination + * protocol between here and the parsec_gpu_data_stage_in, where the readers don't necessarily + * always remove the data from the LRU. + */ + if( 0 != lru_gpu_elem->readers ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Drop LRU-retrieved LEVEL_ZERO copy %p [readers %d, ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->readers, lru_gpu_elem->super.super.obj_reference_count, lru_gpu_elem->original); + goto find_another_data; // TODO: add an assert of some sort to check for leaks here? + } + /* It's also possible that the ref_count of that element is bigger than 1 + * In that case, it's because some task completion did not execute yet, and + * we need to keep it in the list until it reaches 1. + */ + if( lru_gpu_elem->super.super.obj_reference_count > 1 ) { + /* It's also possible (although unlikely) that we livelock here: + * if gpu_mem_lru has *only* elements with readers == 0 but + * ref_count > 1, then we might pop/push forever, and we need + * to make progress on something else to get ref_count == 1 && readers == 0. + * So, we return that there is no more free to do. + * To detect that, we use the first one we push back: if we see it + * again, we're cycling. */ + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Push back LRU-retrieved LEVEL_ZERO copy %p [readers %d, ref_count %d] original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->readers, lru_gpu_elem->super.super.obj_reference_count, lru_gpu_elem->original); + assert(0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == gpu_mem_lru_cycling ) { + gpu_mem_lru_cycling = lru_gpu_elem; + goto find_another_data; + } else { + if( gpu_mem_lru_cycling == lru_gpu_elem ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Cycle detected on allocating memory for %s", + gpu_device->super.name, task_name); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } else { + goto find_another_data; + } + } + } + + /* Make sure the new GPU element is clean and ready to be used */ + assert( master != lru_gpu_elem->original ); + if ( NULL != lru_gpu_elem->original ) { + /* Let's check we're not trying to steal one of our own data */ + oldmaster = lru_gpu_elem->original; + if( !parsec_atomic_trylock( &oldmaster->lock ) ) { + /* Even if we have the lock on oldmaster, any other thread + * might be adding/removing other elements to the list, so we + * need to protect all accesses to gpu_mem_lru with the locked version */ + assert(0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == gpu_mem_lru_cycling ) { + gpu_mem_lru_cycling = lru_gpu_elem; + goto find_another_data; + } else { + if( gpu_mem_lru_cycling == lru_gpu_elem ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Cycle detected on allocating memory for %s", + gpu_device->super.name, task_name); + parsec_atomic_unlock(&master->lock); + return PARSEC_HOOK_RETURN_NEXT; + } else { + goto find_another_data; + } + } + } + for( j = 0; j < i; j++ ) { + if( NULL == this_task->data[j].data_in ) continue; + if( this_task->data[j].data_in->original == oldmaster ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Drop LRU-retrieved LEVEL_ZERO copy %p [ref_count %d] already in use by same task %d:%d original %p", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count, i, j, lru_gpu_elem->original); + /* If we are the owner of this tile we need to make sure it remains available for + * other tasks or we run in deadlock situations. + */ + if( temp_loc[j] != lru_gpu_elem ) + temp_loc[j] = lru_gpu_elem; +#if defined(PARSEC_DEBUG_NOISIER) + /* Make sure the data copy is indeed referenced from the current task */ + for( j = 0; j < i; j++ ) { + if( lru_gpu_elem == temp_loc[j] ) break; + } + assert( j < i ); +#endif /* defined(PARSEC_DEBUG_NOISIER) */ + parsec_atomic_unlock( &oldmaster->lock ); + goto find_another_data; + } + } + if( lru_gpu_elem->readers != 0 ) { + /* Damn, another thread started to use this data. */ + parsec_atomic_unlock( &oldmaster->lock ); + goto find_another_data; + } + int do_unlock = oldmaster->super.obj_reference_count != 1; + parsec_data_copy_detach(oldmaster, lru_gpu_elem, gpu_device->super.device_index); + if( do_unlock ) + parsec_atomic_unlock( &oldmaster->lock ); + assert(lru_gpu_elem->readers == 0); + /* The data is not used, it's not one of ours, and it has been detached from the device + * so no other device can use it as a source for their copy : we can free it or reuse it */ + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\ttask %s:%d repurpose copy %p [ref_count %d] to data %p instead of %p", + gpu_device->super.name, task_name, this_task->task_class->name, i, lru_gpu_elem, + lru_gpu_elem->super.super.obj_reference_count, master, oldmaster); + } + else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s:\ttask %s:%d found detached memory from previously destructed data %p", + gpu_device->super.name, task_name, this_task->task_class->name, i, lru_gpu_elem); + oldmaster = NULL; + } +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + /* Let's free this space, and try again to malloc some space */ + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Release LEVEL_ZERO copy %p (device_ptr %p) [ref_count %d: must be 1], attached to %p", + gpu_device->super.name, + lru_gpu_elem, lru_gpu_elem->device_private, lru_gpu_elem->super.super.obj_reference_count, + oldmaster); +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_free_memory_key, (int64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_use_memory_key_end, + (uint64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, NULL, 0); + } +#endif + assert( 0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + zone_free( gpu_device->memory, (void*)(lru_gpu_elem->device_private) ); + lru_gpu_elem->device_private = NULL; + data_avail_epoch++; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:%s: Release LRU-retrieved LEVEL_ZERO copy %p [ref_count %d: must be 1]", + gpu_device->super.name, task_name, + lru_gpu_elem, lru_gpu_elem->super.super.obj_reference_count); + PARSEC_OBJ_RELEASE(lru_gpu_elem); + assert( NULL == lru_gpu_elem ); + goto malloc_data; + } + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s] Succeeded Allocating LEVEL_ZERO copy %p at real address %p [ref_count %d] for data %p", + gpu_device->super.name, + gpu_elem, gpu_elem->device_private, gpu_elem->super.super.obj_reference_count, master); +#if defined(PARSEC_PROF_TRACE) + if((parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_allocate_memory_key, (int64_t)gpu_elem->device_private, + gpu_device->super.device_index, + &gpu_task->flow_nb_elts[i], PARSEC_PROFILING_EVENT_COUNTER|PARSEC_PROFILING_EVENT_HAS_INFO); + } +#endif +#else + gpu_elem = lru_gpu_elem; +#endif + assert( 0 == gpu_elem->readers ); + gpu_elem->coherency_state = PARSEC_DATA_COHERENCY_INVALID; + gpu_elem->version = 0; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] gets created with version 0 at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, + __FILE__, __LINE__); + parsec_data_copy_attach(master, gpu_elem, gpu_device->super.device_index); + this_task->data[i].data_out = gpu_elem; + /* set the new datacopy type to the correct one */ + this_task->data[i].data_out->dtt = this_task->data[i].data_in->dtt; + temp_loc[i] = gpu_elem; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:%s: Retain and insert LEVEL_ZERO copy %p [ref_count %d] in LRU", + gpu_device->super.name, task_name, + gpu_elem, gpu_elem->super.super.obj_reference_count); + assert(0 != (gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_elem); + parsec_atomic_unlock(&master->lock); + } + if( data_avail_epoch ) { + gpu_device->data_avail_epoch++; + } + return PARSEC_HOOK_RETURN_DONE; +} + +/* Default stage_in function to transfer data to the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_level_zero_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + ze_result_t ret; + parsec_data_copy_t * copy_in; + parsec_data_copy_t * copy_out; + parsec_task_t *task = gtask->ec; + size_t count; + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t *)gpu_stream; + int i; + for(i = 0; i < task->task_class->nb_flows; i++){ + if(flow_mask & (1U << i)){ + copy_in = task->data[i].data_in; + copy_out = task->data[i].data_out; + count = (copy_in->original->nb_elts <= copy_out->original->nb_elts) ? + copy_in->original->nb_elts : copy_out->original->nb_elts; + ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, + copy_out->device_private, + copy_in->device_private, + count, + NULL, + 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", ret, { return PARSEC_ERROR; } ); + } + } + return PARSEC_SUCCESS; +} + +/* Default stage_out function to transfer data from the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_level_zero_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) +{ + ze_result_t ret; + parsec_data_copy_t * copy_in; + parsec_data_copy_t * copy_out; + parsec_task_t *task = gtask->ec; + size_t count; + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t*)gpu_stream; + int i; + for(i = 0; i < task->task_class->nb_flows; i++){ + if(flow_mask & (1U << i)){ + copy_in = task->data[i].data_out; + copy_out = copy_in->original->device_copies[0]; + count = (copy_in->original->nb_elts <= copy_out->original->nb_elts) ? copy_in->original->nb_elts : + copy_out->original->nb_elts; + ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, + copy_out->device_private, + copy_in->device_private, + count, + NULL, + 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", ret, { return PARSEC_ERROR; } ); + } + } + return PARSEC_SUCCESS; +} + +/** + * If the most current version of the data is not yet available on the GPU memory + * schedule a transfer. + * Returns: + * 0: The most recent version of the data is already available on the GPU + * 1: A copy has been scheduled on the corresponding stream + * -1: A copy cannot be issued due to LEVEL_ZERO. + */ +static inline int +parsec_gpu_data_stage_in( parsec_device_level_zero_module_t* level_zero_device, + const parsec_flow_t *flow, + parsec_data_pair_t* task_data, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream ) +{ + parsec_device_gpu_module_t *gpu_device = &level_zero_device->super; + int32_t type = flow->flow_flags; + parsec_data_copy_t* in_elem = task_data->data_in; + parsec_data_copy_t* release_after_data_in_is_attached = NULL; + parsec_data_t* original = in_elem->original; + parsec_gpu_data_copy_t* gpu_elem = task_data->data_out; + uint32_t nb_elts = gpu_task->flow_nb_elts[flow->flow_index]; + int transfer_from = -1; + int undo_readers_inc_if_no_transfer = 0; + + if( gpu_task->task_type == GPU_TASK_TYPE_PREFETCH ) { + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: Prefetch task %p is staging in", + gpu_device->super.name, gpu_task); + } + + parsec_atomic_lock( &original->lock ); + + /** + * If the data will be accessed in write mode, remove it from any GPU data management + * lists until the task is completed. + */ + if( PARSEC_FLOW_ACCESS_WRITE & type ) { + if (gpu_elem->readers > 0 ) { + if( !((1 == gpu_elem->readers) && (PARSEC_FLOW_ACCESS_READ & type)) ) { + parsec_warning("GPU[%s]:\tWrite access to data copy %p [ref_count %d] with existing readers [%d] " + "(possible anti-dependency,\n" + "or concurrent accesses), please prevent that with CTL dependencies\n", + gpu_device->super.name, gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->readers); + parsec_atomic_unlock( &original->lock ); + return -1; + } + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tDetach writable LEVEL_ZERO copy %p [ref_count %d] from any lists", + gpu_device->super.name, gpu_elem, gpu_elem->super.super.obj_reference_count); + /* make sure the element is not in any tracking lists */ + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_elem); + PARSEC_LIST_ITEM_SINGLETON(gpu_elem); + } + + /* Detect if we can do a device to device copy. + * Current limitations: only for read-only data used read-only on the hosting GPU. */ + parsec_device_level_zero_module_t *in_elem_dev = (parsec_device_level_zero_module_t*)parsec_mca_device_get( in_elem->device_index ); + if( (PARSEC_FLOW_ACCESS_READ & type) && !(PARSEC_FLOW_ACCESS_WRITE & type) ) { + int potential_alt_src = 0; + if( PARSEC_DEV_LEVEL_ZERO == in_elem_dev->super.super.type ) { + if( gpu_device->peer_access_mask & (1 << in_elem_dev->level_zero_index) ) { + /* We can directly do D2D, so let's skip the selection */ + goto src_selected; + } + } + + /* If gpu_elem is not invalid, then it is already there and the right version, + * and we're not going to transfer from another source, skip the selection */ + if( gpu_elem->coherency_state != PARSEC_DATA_COHERENCY_INVALID ) + goto src_selected; + + for(int t = 1; t < (int)parsec_nb_devices; t++) { + parsec_device_level_zero_module_t *target = (parsec_device_level_zero_module_t*)parsec_mca_device_get(t); + if( PARSEC_DEV_LEVEL_ZERO != target->super.super.type ) continue; + if(gpu_device->peer_access_mask & (1 << target->level_zero_index)) { + parsec_data_copy_t *candidate = original->device_copies[t]; + if( NULL != candidate && candidate->version == in_elem->version ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is a potential alternative source for in_elem %p on data %p", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index, in_elem, original); + if(PARSEC_DATA_COHERENCY_INVALID == candidate->coherency_state) { + /* We're already pulling this data on candidate... + * If there is another candidate that already has it, we'll use + * that one; otherwise, we'll fall back on the CPU version. */ + potential_alt_src = 1; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is invalid, continuing to look for alternatives", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index); + continue; + } + /* candidate is the best candidate to do D2D. Let's register as a reader for this + * data copy, and we can unlock and schedule the D2D. */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tData copy %p [ref_count %d] on LEVEL_ZERO device %d is the best candidate to to Device to Device copy, increasing its readers to %d", + gpu_device->super.name, candidate, candidate->super.super.obj_reference_count, target->level_zero_index, candidate->readers+1); + parsec_atomic_fetch_inc_int32( &candidate->readers ); + undo_readers_inc_if_no_transfer = 1; + /* We swap data_in with candidate, so we update the reference counters */ + PARSEC_OBJ_RETAIN(candidate); + release_after_data_in_is_attached = task_data->data_in; + task_data->data_in = candidate; + in_elem = candidate; + in_elem_dev = target; + goto src_selected; + } + } + } + if( potential_alt_src ) { + /* We found a potential alternative source, but it's not ready now, + * we delay the scheduling of this task. */ + /** TODO: when considering RW acccesses, don't forget to unchop gpu_elem + * from its queue... */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tThere is a potential alternative source for in_elem %p [ref_count %d] in original %p to go in copy %p [ref_count %d], but it is not ready, falling back on CPU source", + gpu_device->super.name, in_elem, in_elem->super.super.obj_reference_count, original, gpu_elem, gpu_elem->super.super.obj_reference_count); + //return PARSEC_HOOK_RETURN_NEXT; + } + + /* We fall back on the CPU copy */ + assert( original->device_copies[0] != NULL && in_elem->version == original->device_copies[0]->version ); + in_elem = original->device_copies[0]; + assert(task_data->data_in == in_elem); + assert(in_elem->device_index == 0); + in_elem_dev = (parsec_device_level_zero_module_t*)parsec_mca_device_get(in_elem->device_index); + } + + src_selected: + transfer_from = parsec_data_start_transfer_ownership_to_copy(original, gpu_device->super.device_index, (uint8_t)type); + + if( PARSEC_FLOW_ACCESS_WRITE & type && gpu_task->task_type != GPU_TASK_TYPE_PREFETCH ) { + gpu_elem->version++; /* on to the next version */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] increments version to %d at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->version, + __FILE__, __LINE__); + } + + gpu_device->super.required_data_in += nb_elts; + if( -1 != transfer_from ) { + /* If it is already under transfer, don't schedule the transfer again. + * This happens if the task refers twice (or more) to the same input flow */ + if( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_UNDER_TRANSFER ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tMove %s data copy %p [ref_count %d, key %x] of %d bytes\t(src dev: %d, v:%d, ptr:%p, copy:%p [ref_count %d] / dst dev: %d, v:%d, ptr:%p): data copy is already under transfer, ignoring double request", + gpu_device->super.name, + in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D", + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem_dev->super.super.device_index, in_elem->version, (void*) + in_elem->device_private, in_elem, in_elem->super.super.obj_reference_count, + gpu_device->super.device_index, gpu_elem->version, (void*) + gpu_elem->device_private); + } else { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tMove %s data copy %p [ref_count %d, key %x] of %d bytes\t(src dev: %d, v:%d, ptr:%p, copy:%p [ref_count %d] / dst dev: %d, v:%d, ptr:%p)", + gpu_device->super.name, + in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D", + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem_dev->super.super.device_index, in_elem->version, (void*)in_elem->device_private, in_elem, in_elem->super.super.obj_reference_count, + gpu_device->super.device_index, gpu_elem->version, (void*)gpu_elem->device_private); + + assert((gpu_elem->version < in_elem->version) || (gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_NOT_TRANSFER)); + +#if defined(PARSEC_PROF_TRACE) + if( gpu_stream->prof_event_track_enable ) { + parsec_profile_data_collection_info_t info; + + if( NULL != original->dc ) { + info.desc = original->dc; + info.data_id = original->key; + } else { + assert( GPU_TASK_TYPE_PREFETCH != gpu_task->task_type ); + info.desc = (parsec_dc_t*)original; + info.data_id = -1; + } + gpu_task->prof_key_end = -1; + + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type && (parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_PREFETCH) ) { + gpu_task->prof_key_end = parsec_gpu_prefetch_key_end; + gpu_task->prof_event_id = (int64_t)gpu_elem->device_private; + gpu_task->prof_tp_id = level_zero_device->level_zero_index; + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_prefetch_key_start, + gpu_task->prof_event_id, + gpu_task->prof_tp_id, + &info); + } + if(GPU_TASK_TYPE_PREFETCH != gpu_task->task_type && (parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN) ) { + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_movein_key_start, + (int64_t)gpu_elem->device_private, + level_zero_device->level_zero_index, + &info); + } + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) { + parsec_device_level_zero_memory_prof_info_t _info; + _info.size = (uint64_t)nb_elts; + _info.data_key = gpu_elem->original->key; + _info.dc_id = (uint64_t)(gpu_elem->original->dc); + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_gpu_use_memory_key_start, (uint64_t)gpu_elem->device_private, + level_zero_device->level_zero_index, &_info, PARSEC_PROFILING_EVENT_HAS_INFO); + } + } +#endif + /* Push data into the GPU from the source device */ + if(PARSEC_SUCCESS != gpu_task->stage_in(gpu_task, (1U << flow->flow_index), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_in"); + if( in_elem_dev->super.super.type != PARSEC_DEV_LEVEL_ZERO ) + parsec_warning("<<%p>> -> <<%p on LEVEL_ZERO device %d>> [%d, H2D]", + in_elem->device_private, gpu_elem->device_private, level_zero_device->level_zero_index, + nb_elts); + else + parsec_warning("<<%p on LEVEL_ZERO device %d>> -> <<%p on LEVEL_ZERO device %d>> [%d, D2D]", + in_elem->device_private, in_elem_dev->level_zero_index, + gpu_elem->device_private, level_zero_device->level_zero_index, + nb_elts); + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + assert(0); + return -1; + } + + if( in_elem_dev->super.super.type != PARSEC_DEV_LEVEL_ZERO ) + gpu_device->super.transferred_data_in += nb_elts; + else + gpu_device->super.d2d_transfer += nb_elts; + if( GPU_TASK_TYPE_KERNEL == gpu_task->task_type ) + gpu_device->super.nb_data_faults += nb_elts; + + /* update the data version in GPU immediately, and mark the data under transfer */ + assert((gpu_elem->version != in_elem->version) || (gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_NOT_TRANSFER)); + gpu_elem->version = in_elem->version; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU copy %p [ref_count %d] gets the same version %d as copy %p [ref_count %d] at %s:%d", + gpu_device->super.name, + gpu_elem, gpu_elem->super.super.obj_reference_count, gpu_elem->version, in_elem, in_elem->super.super.obj_reference_count, + __FILE__, __LINE__); + + gpu_elem->data_transfer_status = PARSEC_DATA_STATUS_UNDER_TRANSFER; + } + gpu_elem->push_task = gpu_task->ec; /* only the task who does the transfer can modify the data status later. */ + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + return 1; + } + if( undo_readers_inc_if_no_transfer ) + parsec_atomic_fetch_dec_int32( &in_elem->readers ); + assert( gpu_elem->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER ); + + parsec_data_end_transfer_ownership_to_copy(original, gpu_device->super.device_index, (uint8_t)type); + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\t\tNO Move %s for data copy %p [ref_count %d, key %x] of %d bytes (host v:%d / device v:%d)", + gpu_device->super.name, + (NULL == in_elem_dev) ? "h2d" : (in_elem_dev->super.super.type == PARSEC_DEV_LEVEL_ZERO ? "D2D": "H2D"), + gpu_elem, gpu_elem->super.super.obj_reference_count, original->key, nb_elts, + in_elem->version, gpu_elem->version); + parsec_atomic_unlock( &original->lock ); + if( NULL != release_after_data_in_is_attached ) + PARSEC_OBJ_RELEASE(release_after_data_in_is_attached); + /* TODO: data keeps the same coherence flags as before */ + return 0; +} + +static parsec_flow_t parsec_level_zero_data_prefetch_flow = { + .name = "FLOW", + .flow_flags = PARSEC_FLOW_ACCESS_READ, + .flow_index = 0, +}; + +static parsec_task_class_t parsec_level_zero_data_prefetch_tc = { + .name = "LEVEL_ZERO PREFETCH", + .flags = 0, + .task_class_id = 0, + .nb_flows = 1, + .nb_parameters = 0, + .nb_locals = 0, + .dependencies_goal = 0, + .params = { NULL, }, + .in = { &parsec_level_zero_data_prefetch_flow, NULL }, + .out = { NULL, }, + .priority = NULL, + .properties = NULL, + .initial_data = NULL, + .final_data = NULL, + .data_affinity = NULL, + .key_generator = NULL, + .key_functions = NULL, + .make_key = NULL, + .get_datatype = NULL, + .prepare_input = NULL, + .incarnations = NULL, + .prepare_output = NULL, + .find_deps = NULL, + .iterate_successors = NULL, + .iterate_predecessors = NULL, + .release_deps = NULL, + .complete_execution = NULL, + .new_task = NULL, + .release_task = NULL, + .fini = NULL +}; + +static int +parsec_level_zero_destroy_task(parsec_device_gpu_module_t* gpu_device, + parsec_gpu_task_t** out_task) +{ +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + parsec_gpu_task_t *gpu_task = *out_task; + (void)gpu_device; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]: Destroying task %s (%p with ec %p)", + gpu_device->super.name, parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task, gpu_task->ec); + assert( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type || GPU_TASK_TYPE_D2D_COMPLETE == gpu_task->task_type ); + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type) PARSEC_DATA_COPY_RELEASE( gpu_task->ec->data[0].data_in); + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, "GPU[%s]: gpu_task %p freed at %s:%d\n", + gpu_device->super.name, gpu_task, __FILE__, __LINE__); + free( gpu_task->ec ); + gpu_task->ec = NULL; + free( *out_task ); + *out_task = NULL; + return 0; +} + +#if defined(PARSEC_DEBUG_NOISIER) +static char *parsec_level_zero_debug_advice_to_string(int advice) +{ + switch(advice) { + case PARSEC_DEV_DATA_ADVICE_PREFETCH: + return "Prefetch"; + case PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE: + return "Set Preferred Device"; + case PARSEC_DEV_DATA_ADVICE_WARMUP: + return "Mark data as recently used"; + default: + assert(0); + return "Undefined advice"; + } +} +#endif + +static int +parsec_level_zero_data_advise(parsec_device_module_t *dev, parsec_data_t *data, int advice) +{ + parsec_device_gpu_module_t* gpu_device = (parsec_device_gpu_module_t*)dev; +#if defined(PARSEC_DEBUG_NOISIER) + char buffer[64]; + if(NULL != data->dc) { + data->dc->key_to_string(data->dc, data->key, buffer, 64); + } else { + snprintf(buffer, 64, "unbound data"); + } +#endif + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]: User provides advice %s of %s (%p)", + gpu_device->super.name, + parsec_level_zero_debug_advice_to_string(advice), + buffer, + data); + + switch(advice) { + case PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE: + data->preferred_device = dev->device_index; + /* We continue on to the next case, as we want to also + * prefetch the data on the target device, if it is the + * preferred device */ + break; //__attribute__ ((fallthrough)); + case PARSEC_DEV_DATA_ADVICE_PREFETCH: + { + if( parsec_type_contiguous(data->device_copies[ data->owner_device ]->dtt) != PARSEC_SUCCESS){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + " PARSEC_DEV_DATA_ADVICE_PREFETCH cannot be applied to non contiguous types "); + return PARSEC_ERROR; + } + parsec_gpu_task_t* gpu_task = NULL; + gpu_task = (parsec_gpu_task_t*)calloc(1, sizeof(parsec_gpu_task_t)); + gpu_task->task_type = GPU_TASK_TYPE_PREFETCH; + gpu_task->ec = calloc(1, sizeof(parsec_task_t)); + PARSEC_OBJ_CONSTRUCT(gpu_task->ec, parsec_task_t); + gpu_task->ec->task_class = &parsec_level_zero_data_prefetch_tc; + gpu_task->flow[0] = &parsec_level_zero_data_prefetch_flow; + gpu_task->flow_nb_elts[0] = data->device_copies[ data->owner_device ]->original->nb_elts; + gpu_task->stage_in = parsec_level_zero_stage_in; + gpu_task->stage_out = parsec_level_zero_stage_out; + PARSEC_DEBUG_VERBOSE(20, parsec_debug_output, "Retain data copy %p [ref_count %d] at %s:%d", + data->device_copies[ data->owner_device ], + data->device_copies[ data->owner_device ]->super.super.obj_reference_count, + __FILE__, __LINE__); + PARSEC_OBJ_RETAIN(data->device_copies[ data->owner_device ]); + gpu_task->ec->data[0].data_in = data->device_copies[ data->owner_device ]; + gpu_task->ec->data[0].data_out = NULL; + gpu_task->ec->data[0].source_repo_entry = NULL; + gpu_task->ec->data[0].source_repo = NULL; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: data copy %p [ref_count %d] linked to prefetch gpu task %p on GPU copy %p [ref_count %d]", + gpu_device->super.name, gpu_task->ec->data[0].data_in, gpu_task->ec->data[0].data_in->super.super.obj_reference_count, + gpu_task, gpu_task->ec->data[0].data_out, gpu_task->ec->data[0].data_out->super.super.obj_reference_count); + parsec_fifo_push( &(gpu_device->pending), (parsec_list_item_t*)gpu_task ); + return PARSEC_SUCCESS; + } + break; + case PARSEC_DEV_DATA_ADVICE_WARMUP: + return PARSEC_ERR_NOT_IMPLEMENTED; + break; + default: + assert(0); + return PARSEC_ERR_NOT_FOUND; + } + return PARSEC_SUCCESS; +} + +#if PARSEC_GPU_USE_PRIORITIES + +static inline parsec_list_item_t* parsec_push_task_ordered( parsec_list_t* list, + parsec_list_item_t* elem ) +{ + parsec_list_push_sorted(list, elem, parsec_execution_context_priority_comparator); + return elem; +} +#define PARSEC_PUSH_TASK parsec_push_task_ordered +#else +#define PARSEC_PUSH_TASK parsec_list_push_back +#endif + +static parsec_flow_t parsec_level_zero_d2d_complete_flow = { + .name = "FLOW", + .flow_flags = PARSEC_FLOW_ACCESS_READ, + .flow_index = 0, +}; + +static parsec_task_class_t parsec_level_zero_d2d_complete_tc = { + .name = "D2D TRANSFER COMPLETE", + .flags = 0, + .task_class_id = 0, + .nb_flows = 1, + .nb_parameters = 0, + .nb_locals = 0, + .dependencies_goal = 0, + .params = { NULL, }, + .in = { &parsec_level_zero_d2d_complete_flow, NULL }, + .out = { NULL, }, + .priority = NULL, + .properties = NULL, + .initial_data = NULL, + .final_data = NULL, + .data_affinity = NULL, + .key_generator = NULL, + .key_functions = NULL, + .make_key = NULL, + .get_datatype = NULL, + .prepare_input = NULL, + .incarnations = NULL, + .prepare_output = NULL, + .find_deps = NULL, + .iterate_successors = NULL, + .iterate_predecessors = NULL, + .release_deps = NULL, + .complete_execution = NULL, + .new_task = NULL, + .release_task = NULL, + .fini = NULL +}; + +static void +parsec_gpu_send_transfercomplete_cmd_to_device(parsec_data_copy_t *copy, + parsec_device_module_t *current_dev, + parsec_device_module_t *dst_dev) +{ + parsec_gpu_task_t* gpu_task = NULL; + gpu_task = (parsec_gpu_task_t*)calloc(1, sizeof(parsec_gpu_task_t)); + gpu_task->task_type = GPU_TASK_TYPE_D2D_COMPLETE; + gpu_task->ec = calloc(1, sizeof(parsec_task_t)); + PARSEC_OBJ_CONSTRUCT(gpu_task->ec, parsec_task_t); + gpu_task->ec->task_class = &parsec_level_zero_d2d_complete_tc; + gpu_task->flow[0] = &parsec_level_zero_d2d_complete_flow; + gpu_task->flow_nb_elts[0] = copy->original->nb_elts; + gpu_task->stage_in = parsec_level_zero_stage_in; + gpu_task->stage_out = parsec_level_zero_stage_out; + gpu_task->ec->data[0].data_in = copy; /* We need to set not-null in data_in, so that the fake flow is + * not ignored when poping the data from the fake task */ + gpu_task->ec->data[0].data_out = copy; /* We "free" data[i].data_out if its readers reaches 0 */ + gpu_task->ec->data[0].source_repo_entry = NULL; + gpu_task->ec->data[0].source_repo = NULL; +#if defined(PARSEC_PROF_TRACE) + gpu_task->prof_key_end = -1; /* D2D complete tasks are pure internal management, we do not trace them */ +#endif + (void)current_dev; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: data copy %p [ref_count %d] D2D transfer is complete, sending order to count it " + "to LEVEL_ZERO Device %s", + current_dev->name, gpu_task->ec->data[0].data_out, + gpu_task->ec->data[0].data_out->super.super.obj_reference_count, + dst_dev->name); + parsec_fifo_push( &(((parsec_device_gpu_module_t*)dst_dev)->pending), (parsec_list_item_t*)gpu_task ); +} + +static int +parsec_gpu_callback_complete_push(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t **gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + parsec_device_level_zero_module_t* level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + (void)gpu_stream; + + parsec_gpu_task_t *gtask = *gpu_task; + parsec_task_t *task; + int32_t i; +#if defined(PARSEC_DEBUG_NOISIER) + char task_str[MAX_TASK_STRLEN]; + char task_str2[MAX_TASK_STRLEN]; +#endif + const parsec_flow_t *flow; + /** + * Even though level_zero event return success, the PUSH may not be + * completed if no PUSH is required by this task and the PUSH is + * actually done by another task, so we need to check if the data is + * actually ready to use + */ + assert(gpu_stream == gpu_device->exec_stream[0]); + task = gtask->ec; + PARSEC_DEBUG_VERBOSE(19, parsec_gpu_output_stream, + "GPU[%s]: parsec_gpu_callback_complete_push, PUSH of %s", + gpu_device->super.name, parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task)); + + for( i = 0; i < task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == task->data[i].data_in ) continue; + /* We also don't push back non-parsec-owned copies */ + if(NULL != task->data[i].data_out && + 0 == (task->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED)) continue; + + flow = gtask->flow[i]; + assert( flow ); + assert( flow->flow_index == i ); + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + if(task->data[i].data_out->push_task == task ) { /* only the task who did this PUSH can modify the status */ + parsec_atomic_lock(&task->data[i].data_out->original->lock); + task->data[i].data_out->data_transfer_status = PARSEC_DATA_STATUS_COMPLETE_TRANSFER; + parsec_data_end_transfer_ownership_to_copy(task->data[i].data_out->original, + gpu_device->super.device_index, + flow->flow_flags); +#if defined(PARSEC_PROF_TRACE) + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_IN) { + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_movein_key_end, + (int64_t)(int64_t)task->data[i].data_out->device_private, + level_zero_device->level_zero_index, + NULL); + } +#endif + task->data[i].data_out->push_task = NULL; + parsec_atomic_unlock(&task->data[i].data_out->original->lock); + parsec_device_gpu_module_t *src_device = + (parsec_device_gpu_module_t*)parsec_mca_device_get( task->data[i].data_in->device_index ); + if( PARSEC_DEV_LEVEL_ZERO == src_device->super.type ) { + int om; + while(1) { + /* There are two ways out: + * either we exit with om = 0, and then nobody was managing src_device, + * and nobody can start managing src_device until we make it change from -1 to 0 + * (but anybody who has work to do will wait until that happens), or + * we exit with om > 0, then there is a manager for that thread, and we have + * increased mutex to warn the manager that there is another task for it to do. + */ + om = src_device->mutex; + if(om == 0) { + /* Nobody at the door, let's try to lock the door */ + if( parsec_atomic_cas_int32(&src_device->mutex, 0, -1) ) + break; + continue; + } + if(om < 0 ) { + /* Damn, another thread is also trying to do an atomic operation on src_device, + * we give it some time and try again */ + struct timespec delay; + delay.tv_nsec = 100; + delay.tv_sec = 0; + nanosleep(&delay, NULL); + continue; + } + /* There is a manager, let's try to reserve another task to do. + * If that fails, the manager may have leaved, try a gain. */ + if( parsec_atomic_cas_int32(&src_device->mutex, om, om+1) ) + break; + } + if( 0 == om ) { + int rc; + /* Nobody is at the door to handle that event on the source of that data... + * we do the command directly */ + parsec_atomic_lock( &task->data[i].data_in->original->lock ); + task->data[i].data_in->readers--; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tExecuting D2D transfer complete for copy %p [ref_count %d] for " + "device %s -- readers now %d", + gpu_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count, src_device->super.name, + task->data[i].data_in->readers); + assert(task->data[i].data_in->readers >= 0); + if(0 == task->data[i].data_in->readers) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tMake read-only copy %p [ref_count %d] available", + gpu_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count); + parsec_list_item_ring_chop((parsec_list_item_t*)task->data[i].data_in); + PARSEC_LIST_ITEM_SINGLETON(task->data[i].data_in); + parsec_list_push_back(&src_device->gpu_mem_lru, (parsec_list_item_t*)task->data[i].data_in); + src_device->data_avail_epoch++; + } + parsec_atomic_unlock( &task->data[i].data_in->original->lock ); + /* Notify any waiting thread that we're done messing with that device structure */ + rc = parsec_atomic_cas_int32(&src_device->mutex, -1, 0); (void)rc; + assert(rc); + } else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tSending D2D transfer complete command to %s for copy %p " + "[ref_count %d] -- readers is still %d", + gpu_device->super.name, src_device->super.name, task->data[i].data_in, + task->data[i].data_in->super.super.obj_reference_count, task->data[i].data_in->readers); + parsec_gpu_send_transfercomplete_cmd_to_device(task->data[i].data_in, + (parsec_device_module_t*)gpu_device, + (parsec_device_module_t*)src_device); + } + } + continue; + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tparsec_gpu_callback_complete_push, PUSH of %s: task->data[%d].data_out = %p [ref_count = %d], and push_task is %s, %s because transfer_status is %d", + gpu_device->super.name, parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task), + i, task->data[i].data_out, task->data[i].data_out->super.super.obj_reference_count, + (NULL != task->data[i].data_out->push_task) ? parsec_task_snprintf(task_str2, MAX_TASK_STRLEN, task->data[i].data_out->push_task) : "(null)", + (task->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER) ? "all is good" : "Assertion", + task->data[i].data_out->data_transfer_status); + assert(task->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER); + if( task->data[i].data_out->data_transfer_status != PARSEC_DATA_STATUS_COMPLETE_TRANSFER ) { /* data is not ready */ + /** + * As long as we have only one stream to push the data on the GPU we should never + * end up in this case. Remove previous assert if changed. + */ + return -1; + } + } + gtask->complete_stage = NULL; + + if( GPU_TASK_TYPE_PREFETCH == gtask->task_type ) { + parsec_data_copy_t *gpu_copy = task->data[0].data_out; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + assert(NULL != gpu_copy); + if( NULL != gpu_copy->original->dc ) + gpu_copy->original->dc->key_to_string(gpu_copy->original->dc, gpu_copy->original->key, tmp, MAX_TASK_STRLEN); + else + snprintf(tmp, MAX_TASK_STRLEN, "unbound data"); +#endif + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:\tPrefetch for data copy %p [ref_count %d] (%s) done. readers = %d, device_index = %d, version = %d, flags = %d, state = %d, data_transfer_status = %d", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, + tmp, + gpu_copy->readers, gpu_copy->device_index, gpu_copy->version, + gpu_copy->flags, gpu_copy->coherency_state, gpu_copy->data_transfer_status); + gpu_copy->readers--; + if( 0 == gpu_copy->readers ) { + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]:\tMake copy %p [ref_count %d] available after prefetch from gpu_task %p, ec %p", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, gtask, gtask->ec); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } + return parsec_level_zero_destroy_task(gpu_device, gpu_task); + } + + return 0; +} + +/** + * This function tries to progress a stream, by picking up a ready task + * and applying the progress function. The task to be progresses is + * always the highest priority in the waiting queue, even when a task + * has been specified as an input argument. + * The progress function is either specified by the caller via the + * upstream_progress_fct input argument or by the next task to be progresses + * via the submit function associated with the task. In any case, this + * function progresses a single task, which is then returned as the + * out_task parameter. + * + * Beware: this function does not generate errors by itself, instead + * it propagates upward the return code of the progress function. + * However, by convention the error code follows the parsec_hook_return_e + * enum. + */ +static inline int +progress_stream( parsec_device_gpu_module_t* gpu_device, + parsec_gpu_exec_stream_t* stream, + advance_task_function_t upstream_progress_fct, + parsec_gpu_task_t* task, + parsec_gpu_task_t** out_task ) +{ + advance_task_function_t progress_fct; + int saved_rc = 0, rc; +#if defined(PARSEC_DEBUG_NOISIER) + char task_str[MAX_TASK_STRLEN]; +#endif + parsec_level_zero_exec_stream_t *level_zero_stream = (parsec_level_zero_exec_stream_t *)stream; + + /* We always handle the tasks in order. Thus if we got a new task, add it to the + * local list (possibly by reordering the list). Also, as we can return a single + * task first try to see if anything completed. */ + if( NULL != task ) { + PARSEC_PUSH_TASK(stream->fifo_pending, (parsec_list_item_t*)task); + task = NULL; + } + *out_task = NULL; + progress_fct = upstream_progress_fct; + + if( NULL != stream->tasks[stream->end] ) { + rc = zeEventQueryStatus(level_zero_stream->events[stream->end]); + if( ZE_RESULT_SUCCESS == rc ) { + /* Save the task for the next step */ + task = *out_task = stream->tasks[stream->end]; + PARSEC_DEBUG_VERBOSE(19, parsec_gpu_output_stream, + "GPU[%s]: Completed %s priority %d on stream %s{%p}", + gpu_device->super.name, + parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task->ec), + task->ec->priority, stream->name, (void*)stream); + stream->tasks[stream->end] = NULL; + stream->end = (stream->end + 1) % stream->max_events; + +#if defined(PARSEC_PROF_TRACE) + if( stream->prof_event_track_enable ) { + if( task->prof_key_end != -1 ) { + PARSEC_PROFILING_TRACE(stream->profiling, task->prof_key_end, task->prof_event_id, task->prof_tp_id, NULL); + } + } +#endif /* (PARSEC_PROF_TRACE) */ + + rc = PARSEC_HOOK_RETURN_DONE; + if (task->complete_stage) + rc = task->complete_stage(gpu_device, out_task, stream); + /* the task can be withdrawn by the system */ + return rc; + } + if( ZE_RESULT_NOT_READY != rc ) { + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeEventQueryStatus ", rc, + {return PARSEC_HOOK_RETURN_AGAIN;} ); + } + } + + grab_a_task: + if( NULL == stream->tasks[stream->start] ) { /* there is room on the stream */ + task = (parsec_gpu_task_t*)parsec_list_pop_front(stream->fifo_pending); /* get the best task */ + } + if( NULL == task ) { /* No tasks, we're done */ + return saved_rc; + } + PARSEC_LIST_ITEM_SINGLETON((parsec_list_item_t*)task); + + assert( NULL == stream->tasks[stream->start] ); + /** + * In case the task is succesfully progressed, the corresponding profiling + * event is triggered. + */ + if ( NULL == upstream_progress_fct ) { + /* Grab the submit function */ + progress_fct = task->submit; +#if defined(PARSEC_DEBUG_PARANOID) + int i; + const parsec_flow_t *flow; + for( i = 0; i < task->ec->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == task->ec->data[i].data_in ) continue; + + flow = task->flow[i]; + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + if( 0 == (task->ec->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + assert(task->ec->data[i].data_out->data_transfer_status == PARSEC_DATA_STATUS_COMPLETE_TRANSFER); + } +#endif /* defined(PARSEC_DEBUG_PARANOID) */ + } + rc = progress_fct( gpu_device, task, stream ); + if( 0 > rc ) { + if( PARSEC_HOOK_RETURN_AGAIN != rc && + PARSEC_HOOK_RETURN_ASYNC != rc ) { + *out_task = task; + return rc; + } + + if( PARSEC_HOOK_RETURN_ASYNC == rc ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: GPU task %p has been removed by the progress function", + gpu_device->super.name, (void*)task); + } else { + parsec_fifo_push(stream->fifo_pending, (parsec_list_item_t*)task); + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, + "GPU[%s]: Reschedule task %p: no room available on the GPU for data", + gpu_device->super.name, (void*)task->ec); + } + *out_task = NULL; + return PARSEC_HOOK_RETURN_DONE; + } + /** + * Do not skip the level_zero event generation. The problem is that some of the inputs + * might be in the pipe of being transferred to the GPU. If we activate this task + * too early, it might get executed before the data is available on the GPU. + * Obviously, this lead to incorrect results. + */ + rc = zeCommandListAppendSignalEvent( level_zero_stream->level_zero_cl, level_zero_stream->events[stream->start] ); + assert(ZE_RESULT_SUCCESS == rc); + stream->tasks[stream->start] = task; + stream->start = (stream->start + 1) % stream->max_events; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]: Submitted %s(task %p) priority %d on stream %s{%p}", + gpu_device->super.name, + task->ec->task_class->name, (void*)task->ec, task->ec->priority, + stream->name, (void*)stream); + + task = NULL; + goto grab_a_task; +} + +/** + * This function schedule the move of all the data required for a + * specific task from the main memory into the GPU memory. + * + * Returns: + * a positive number: the number of data to be moved. + * -1: data cannot be moved into the GPU. + * -2: No more room on the GPU to move this data. + */ +static int +parsec_gpu_kernel_push( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + parsec_task_t *this_task = gpu_task->ec; + const parsec_flow_t *flow; + int i, ret = 0; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + +#if 0 + if( gpu_task->last_data_check_epoch == gpu_device->data_avail_epoch ) + return PARSEC_HOOK_RETURN_AGAIN; +#endif + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Try to Push %s", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task) ); + + if( GPU_TASK_TYPE_PREFETCH == gpu_task->task_type ) { + if( NULL == gpu_task->ec->data[0].data_in->original ) { + /* The PREFETCH order comes after the copy was detached and released, ignore it */ + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: %s has been released already, destroying prefetch request", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task)); + parsec_level_zero_destroy_task(gpu_device, &gpu_task); + return PARSEC_HOOK_RETURN_ASYNC; + } + if( NULL != gpu_task->ec->data[0].data_in->original->device_copies[gpu_device->super.device_index] && + gpu_task->ec->data[0].data_in->original->owner_device == gpu_device->super.device_index ) { + /* There is already a copy of this data in the GPU */ + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream, + "GPU[%s]: %s data_copy at index %d is %p, destroying prefetch request", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_device->super.device_index, + gpu_task->ec->data[0].data_in->original->device_copies[gpu_device->super.device_index]); + parsec_level_zero_destroy_task(gpu_device, &gpu_task); + return PARSEC_HOOK_RETURN_ASYNC; + } + } + + /* Do we have enough available memory on the GPU to hold the input and output data ? */ + ret = parsec_gpu_data_reserve_device_space( level_zero_device, gpu_task ); + if( ret < 0 ) { + gpu_task->last_data_check_epoch = gpu_device->data_avail_epoch; + return ret; + } + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + + flow = gpu_task->flow[i]; + /* Skip CTL flows */ + if(PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags)) continue; + + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* If there is already a GPU data copy (set by reserve_device_space), and this copy + * is not parsec-owned, don't stage in */ + if( NULL != this_task->data[i].data_out && + (0 == (this_task->data[i].data_out->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) ) continue; + + assert( NULL != parsec_data_copy_get_ptr(this_task->data[i].data_in) ); + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\t\tIN Data of %s <%x> on GPU", + gpu_device->super.name, flow->name, + this_task->data[i].data_out->original->key); + ret = parsec_gpu_data_stage_in( level_zero_device, flow, + &(this_task->data[i]), gpu_task, gpu_stream ); + if( ret < 0 ) { + return ret; + } + } + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Push task %s DONE", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); + gpu_task->complete_stage = parsec_gpu_callback_complete_push; +#if defined(PARSEC_PROF_TRACE) + gpu_task->prof_key_end = -1; /* We do not log that event as the completion of this task */ +#endif + return ret; +} + +/** + * This function schedule the move of all the modified data for a + * specific task from the GPU memory into the main memory. + * + * Returns: negative number if any error occured. + * positive: the number of data to be moved. + */ +static int +parsec_gpu_kernel_pop( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy; + parsec_data_t *original; + uint32_t nb_elts; + const parsec_flow_t *flow; + int return_code = 0, how_many = 0, i, update_data_epoch = 0; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + + if (gpu_task->task_type == GPU_TASK_TYPE_D2HTRANSFER) { + for( i = 0; i < this_task->locals[0].value; i++ ) { + gpu_copy = this_task->data[i].data_out; + /* If the gpu copy is not owned by parsec, we don't manage it at all */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + original = gpu_copy->original; + if(PARSEC_SUCCESS != gpu_task->stage_out(gpu_task, (1U << i), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_out from device "); + parsec_warning("data %s <<%p>> -> <<%p>>\n", this_task->task_class->out[i]->name, + gpu_copy->device_private, original->device_copies[0]->device_private); + return_code = -2; + goto release_and_return_error; + } + } + return return_code; + } + + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Try to Pop %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* We need to manage all data that has been used as input, even if they were read only */ + + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + flow = gpu_task->flow[i]; + if( PARSEC_FLOW_ACCESS_NONE == (PARSEC_FLOW_ACCESS_MASK & flow->flow_flags) ) continue; /* control flow */ + + gpu_copy = this_task->data[i].data_out; + + /* If the gpu copy is not owned by parsec, we don't manage it at all */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + + original = gpu_copy->original; + nb_elts = gpu_task->flow_nb_elts[i]; + + assert( this_task->data[i].data_in == NULL || original == this_task->data[i].data_in->original ); + + if( !(flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Do not propagate GPU copies to successors (temporary solution) */ + this_task->data[i].data_out = original->device_copies[0]; + } + parsec_atomic_lock(&original->lock); + if( flow->flow_flags & PARSEC_FLOW_ACCESS_READ ) { + gpu_copy->readers--; + if( gpu_copy->readers < 0 ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: While trying to Pop %s, gpu_copy %p [ref_count %d] on flow %d with original %p had already 0 readers", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task), + gpu_copy, gpu_copy->super.super.obj_reference_count, + i, original); + + } + assert(gpu_copy->readers >= 0); + if( (0 == gpu_copy->readers) && + !(flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tMake read-only copy %p [ref_count %d] available on flow %s", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name); + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); /* TODO: singleton instead? */ + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + update_data_epoch = 1; + parsec_atomic_unlock(&original->lock); + continue; /* done with this element, go for the next one */ + } + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tread copy %p [ref_count %d] on flow %s has readers (%i)", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name, gpu_copy->readers); + } + if( flow->flow_flags & PARSEC_FLOW_ACCESS_WRITE ) { + assert( gpu_copy == parsec_data_get_copy(gpu_copy->original, gpu_device->super.device_index) ); + + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "GPU[%s]:\tOUT Data copy %p [ref_count %d] for flow %s", + gpu_device->super.name, gpu_copy, gpu_copy->super.super.obj_reference_count, flow->name); + + /* Stage the transfer of the data back to main memory */ + gpu_device->super.required_data_out += nb_elts; + assert( ((parsec_list_item_t*)gpu_copy)->list_next == (parsec_list_item_t*)gpu_copy ); + assert( ((parsec_list_item_t*)gpu_copy)->list_prev == (parsec_list_item_t*)gpu_copy ); + + assert( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ); + if( gpu_task->pushout & (1 << i) ) { + /* TODO: make sure no readers are working on the CPU version */ + original = gpu_copy->original; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tMove D2H data <%s:%x> copy %p [ref_count %d] -- D:%p -> H:%p requested", + gpu_device->super.name, flow->name, original->key, gpu_copy, gpu_copy->super.super.obj_reference_count, + (void*)gpu_copy->device_private, original->device_copies[0]->device_private); +#if defined(PARSEC_PROF_TRACE) + if( gpu_stream->prof_event_track_enable ) { + if(parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_DATA_OUT) { + parsec_profile_data_collection_info_t info; + if( NULL != original->dc ) { + info.desc = original->dc; + info.data_id = original->key; + } else { + info.desc = (parsec_dc_t*)original; + info.data_id = -1; + } + gpu_task->prof_key_end = parsec_gpu_moveout_key_end; + gpu_task->prof_tp_id = this_task->taskpool->taskpool_id; + gpu_task->prof_event_id = this_task->task_class->key_functions->key_hash(this_task->task_class->make_key(this_task->taskpool, this_task->locals), NULL); + PARSEC_PROFILING_TRACE(gpu_stream->profiling, + parsec_gpu_moveout_key_start, + gpu_task->prof_event_id, + gpu_task->prof_tp_id, + &info); + } else { + gpu_task->prof_key_end = -1; + } + } +#endif + /* Move the data back into main memory */ + if( PARSEC_SUCCESS != gpu_task->stage_out(gpu_task, (1U << flow->flow_index), gpu_stream)){ + parsec_warning( "%s:%d %s", __FILE__, __LINE__, + "gpu_task->stage_out from device "); + parsec_warning("data %s <<%p>> -> <<%p>>\n", this_task->task_class->out[i]->name, + gpu_copy->device_private, original->device_copies[0]->device_private); + return_code = -2; + parsec_atomic_unlock(&original->lock); + goto release_and_return_error; + } + gpu_device->super.transferred_data_out += nb_elts; /* TODO: not hardcoded, use datatype size */ + how_many++; + } else { + assert( 0 == gpu_copy->readers ); + } + } + parsec_atomic_unlock(&original->lock); + } + + release_and_return_error: + if( update_data_epoch ) { + gpu_device->data_avail_epoch++; + } + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Pop %s DONE (return %d data epoch %"PRIu64")", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task), return_code, gpu_device->data_avail_epoch ); + + return (return_code < 0 ? return_code : how_many); +} + +/** + * Make sure all data on the device is correctly put back into the queues. + */ +static int +parsec_gpu_kernel_epilog( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy, *cpu_copy; + parsec_data_t *original; + int i; + +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Epilog of %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); +#endif + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* Don't bother if there is no real data (aka. CTL or no output) */ + if(NULL == this_task->data[i].data_out) continue; + + + if( !(gpu_task->flow[i]->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Warning data_out for read only flows has been overwritten in pop */ + continue; + } + + gpu_copy = this_task->data[i].data_out; + original = gpu_copy->original; + cpu_copy = original->device_copies[0]; + + /* If it is a copy managed by the user, don't bother either */ + if( 0 == (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) continue; + + /** + * There might be a race condition here. We can't assume the first CPU + * version is the corresponding CPU copy, as a new CPU-bound data + * might have been created meanwhile. + * + * WARNING: For now we always forward the cpu_copy to the next task, to + * do that, we lie to the engine by updating the CPU copy to the same + * status than the GPU copy without updating the data itself. Thus, the + * cpu copy is really invalid. this is related to Issue #88, and the + * fact that: + * - we don't forward the gpu copy as output + * - we always take a cpu copy as input, so it has to be in the + * same state as the GPU to prevent an extra data movement. + */ + assert( PARSEC_DATA_COHERENCY_OWNED == gpu_copy->coherency_state ); + gpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + cpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + + /** + * The cpu_copy will be updated in the completion, and at that moment + * the two versions will be identical. + */ + cpu_copy->version = gpu_copy->version; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: CPU copy %p [ref_count %d] gets the same version %d as GPU copy %p [ref_count %d] at %s:%d", + gpu_device->super.name, + cpu_copy, cpu_copy->super.super.obj_reference_count, cpu_copy->version, gpu_copy, gpu_copy->super.super.obj_reference_count, + __FILE__, __LINE__); + + /** + * Let's lie to the engine by reporting that working version of this + * data (aka. the one that GEMM worked on) is now on the CPU. + */ + this_task->data[i].data_out = cpu_copy; + + assert( 0 == gpu_copy->readers ); + + if( gpu_task->pushout & (1 << i) ) { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the read LRU in %s", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + parsec_list_item_ring_chop((parsec_list_item_t*)gpu_copy); + PARSEC_LIST_ITEM_SINGLETON(gpu_copy); + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } else { + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the owned LRU in %s", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + parsec_list_push_back(&gpu_device->gpu_mem_owned_lru, (parsec_list_item_t*)gpu_copy); + } + } + return 0; +} + +/** @brief Release the LEVEL_ZERO copies of the data used in WRITE mode. + * + * @details This function can be used when the LEVEL_ZERO task didn't run + * to completion on the device (either due to an error, or + * simply because the body requested a reexecution on a + * different location). It releases the LEVEL_ZERO copies of the + * output data, allowing them to be reused by the runtime. + * This function has the drawback of kicking in too late, + * after all data transfers have been completed toward the + * device. + * + * @param [IN] gpu_device, the GPU device the the task has been + * supposed to execute. + * @param [IN] gpu_task, the task that has been cancelled, and which + * needs it's data returned to the runtime. + * @return Currently only success. + */ +static int +parsec_gpu_kernel_cleanout( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task ) +{ + parsec_task_t *this_task = gpu_task->ec; + parsec_gpu_data_copy_t *gpu_copy, *cpu_copy; + parsec_data_t *original; + int i, data_avail_epoch = 0; + +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]: Cleanup of %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, this_task) ); +#endif + + for( i = 0; i < this_task->task_class->nb_flows; i++ ) { + /* Make sure data_in is not NULL */ + if( NULL == this_task->data[i].data_in ) continue; + + /* Don't bother if there is no real data (aka. CTL or no output) */ + if(NULL == this_task->data[i].data_out) continue; + if( !(gpu_task->flow[i]->flow_flags & PARSEC_FLOW_ACCESS_WRITE) ) { + /* Warning data_out for read only flows has been overwritten in pop */ + continue; + } + + gpu_copy = this_task->data[i].data_out; + original = gpu_copy->original; + parsec_atomic_lock(&original->lock); + assert(gpu_copy->super.super.obj_reference_count > 1); + /* Issue #134 */ + parsec_data_copy_detach(original, gpu_copy, gpu_device->super.device_index); + gpu_copy->coherency_state = PARSEC_DATA_COHERENCY_SHARED; + cpu_copy = original->device_copies[0]; + + /** + * Let's lie to the engine by reporting that working version of this + * data (aka. the one that GEMM worked on) is now on the CPU. + */ + this_task->data[i].data_out = cpu_copy; + if( 0 != (gpu_copy->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)gpu_copy); + } + parsec_atomic_unlock(&original->lock); + data_avail_epoch++; + PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, + "LEVEL_ZERO copy %p [ref_count %d] moved to the read LRU in %s\n", + gpu_copy, gpu_copy->super.super.obj_reference_count, __func__); + } + if( data_avail_epoch ) /* Update data availability epoch */ + gpu_device->data_avail_epoch++; + return 0; +} + +/** + * This version is based on 4 streams: one for transfers from the memory to + * the GPU, 2 for kernel executions and one for transfers from the GPU into + * the main memory. The synchronization on each stream is based on LEVEL_ZERO events, + * such an event indicate that a specific epoch of the lifetime of a task has + * been completed. Each type of stream (in, exec and out) has a pending FIFO, + * where tasks ready to jump to the respective step are waiting. + */ +parsec_hook_return_t +parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ) +{ + parsec_device_gpu_module_t* gpu_device; + int rc, exec_stream = 0; + parsec_gpu_task_t *progress_task, *out_task_submit = NULL, *out_task_pop = NULL; +#if defined(PARSEC_DEBUG_NOISIER) + char tmp[MAX_TASK_STRLEN]; +#endif + int pop_null = 0; + + gpu_device = (parsec_device_gpu_module_t*)parsec_mca_device_get(which_gpu); + +#if defined(PARSEC_PROF_TRACE) + PARSEC_PROFILING_TRACE_FLAGS( es->es_profile, + PARSEC_PROF_FUNC_KEY_END(gpu_task->ec->taskpool, + gpu_task->ec->task_class->task_class_id), + gpu_task->ec->task_class->key_functions->key_hash(gpu_task->ec->task_class->make_key(gpu_task->ec->taskpool, gpu_task->ec->locals), NULL), + gpu_task->ec->taskpool->taskpool_id, NULL, + PARSEC_PROFILING_EVENT_RESCHEDULED ); +#endif /* defined(PARSEC_PROF_TRACE) */ + + /* Check the GPU status -- three kinds of values for rc: + * - rc < 0: somebody is doing a short atomic operation while there is no manager, + * so wait. + * - rc == 0: there is no manager, and at the exit of the while, this thread + * made rc go from 0 to 1, so it is the new manager of the GPU and + * needs to deal with gpu_task + * - rc > 0: there is a manager, and at the exit of the while, this thread has + * committed new work that the manager will need to do, but the work is + * not in the queue yet. + */ + while(1) { + rc = gpu_device->mutex; + struct timespec delay; + if( rc >= 0 ) { + if( parsec_atomic_cas_int32( &gpu_device->mutex, rc, rc+1 ) ) { + break; + } + } else { + delay.tv_nsec = 100; + delay.tv_sec = 0; + nanosleep(&delay, NULL); + } + } + if( 0 < rc ) { + parsec_fifo_push( &(gpu_device->pending), (parsec_list_item_t*)gpu_task ); + return PARSEC_HOOK_RETURN_ASYNC; + } + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream,"GPU[%s]: Entering GPU management at %s:%d", + gpu_device->super.name, __FILE__, __LINE__); + +#if defined(PARSEC_PROF_TRACE) + if( parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN ) + PARSEC_PROFILING_TRACE( es->es_profile, parsec_gpu_own_GPU_key_start, + (unsigned long)es, PROFILE_OBJECT_ID_NULL, NULL ); +#endif /* defined(PARSEC_PROF_TRACE) */ + + check_in_deps: + if( NULL != gpu_task ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%s]:\tUpload data (if any) for %s priority %d", + gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task->ec->priority ); + } + rc = progress_stream( gpu_device, + gpu_device->exec_stream[0], + parsec_gpu_kernel_push, + gpu_task, &progress_task ); + if( rc < 0 ) { /* In case of error progress_task is the task that raised it */ + if( -1 == rc ) + goto disable_gpu; + /* We are in the early stages, and if there no room on the GPU for a task we need to + * delay all retries for the same task for a little while. Meanwhile, put the task back + * trigger a device flush, and keep executing tasks that have their data on the device. + */ + if( NULL != progress_task ) { + PARSEC_PUSH_TASK(gpu_device->exec_stream[0]->fifo_pending, (parsec_list_item_t*)progress_task); + progress_task = NULL; + } + /* If we can extract data go for it, otherwise try to drain the pending tasks */ + gpu_task = parsec_gpu_create_W2R_task(gpu_device, es); + if( NULL != gpu_task ) + goto get_data_out_of_device; + } + gpu_task = progress_task; + + /* Stage-in completed for this task: it is ready to be executed */ + exec_stream = (exec_stream + 1) % (gpu_device->max_exec_streams - 2); /* Choose an exec_stream */ + if( NULL != gpu_task ) { + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tExecute %s priority %d", gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec), + gpu_task->ec->priority ); + } + rc = progress_stream( gpu_device, + gpu_device->exec_stream[2+exec_stream], + NULL, + gpu_task, &progress_task ); + if( rc < 0 ) { + if( PARSEC_HOOK_RETURN_DISABLE == rc ) + goto disable_gpu; + if( PARSEC_HOOK_RETURN_ASYNC != rc ) { + /* Reschedule the task. As the chore_id has been modified, + another incarnation of the task will be executed. */ + if( NULL != progress_task ) { + parsec_gpu_kernel_cleanout(gpu_device, progress_task); + __parsec_reschedule(es, progress_task->ec); + gpu_task = progress_task; + progress_task = NULL; + goto remove_gpu_task; + } + gpu_task = NULL; + goto fetch_task_from_shared_queue; + } + progress_task = NULL; + } + gpu_task = progress_task; + out_task_submit = progress_task; + + get_data_out_of_device: + if( NULL != gpu_task ) { /* This task has completed its execution */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tRetrieve data (if any) for %s priority %d", gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec), + gpu_task->ec->priority ); + } + /* Task is ready to move the data back to main memory */ + rc = progress_stream( gpu_device, + gpu_device->exec_stream[1], + parsec_gpu_kernel_pop, + gpu_task, &progress_task ); + if( rc < 0 ) { + if( -1 == rc ) + goto disable_gpu; + } + if( NULL != progress_task ) { + /* We have a succesfully completed task. However, it is not gpu_task, as + * it was just submitted into the data retrieval system. Instead, the task + * ready to move into the next level is the progress_task. + */ + gpu_task = progress_task; + progress_task = NULL; + goto complete_task; + } + gpu_task = progress_task; + out_task_pop = progress_task; + + fetch_task_from_shared_queue: + assert( NULL == gpu_task ); + if (1 == parsec_LEVEL_ZERO_sort_pending_list && out_task_submit == NULL && out_task_pop == NULL) { + parsec_gpu_sort_pending_list(gpu_device); + } + gpu_task = (parsec_gpu_task_t*)parsec_fifo_try_pop( &(gpu_device->pending) ); + if( NULL != gpu_task ) { + pop_null = 0; + gpu_task->last_data_check_epoch = gpu_device->data_avail_epoch - 1; /* force at least one tour */ + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tGet from shared queue %s priority %d", gpu_device->super.name, + parsec_gpu_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task), + gpu_task->ec->priority); + if( GPU_TASK_TYPE_D2D_COMPLETE == gpu_task->task_type ) { + goto get_data_out_of_device; + } + } else { + pop_null++; + if( pop_null % 1024 == 1023 ) { + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream, "GPU[%s]:\tStill waiting for %d tasks to execute, but poped NULL the last %d times I tried to pop something...", + gpu_device->super.name, gpu_device->mutex, pop_null); + } + } + goto check_in_deps; + + complete_task: + assert( NULL != gpu_task ); + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "GPU[%s]:\tComplete %s", + gpu_device->super.name, + parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec)); + /* Everything went fine so far, the result is correct and back in the main memory */ + PARSEC_LIST_ITEM_SINGLETON(gpu_task); + if (gpu_task->task_type == GPU_TASK_TYPE_D2HTRANSFER) { + parsec_gpu_W2R_task_fini(gpu_device, gpu_task, es); + gpu_task = progress_task; + goto fetch_task_from_shared_queue; + } + if (gpu_task->task_type == GPU_TASK_TYPE_D2D_COMPLETE) { + free( gpu_task->ec ); + gpu_task->ec = NULL; + goto remove_gpu_task; + } + parsec_gpu_kernel_epilog( gpu_device, gpu_task ); + __parsec_complete_execution( es, gpu_task->ec ); + gpu_device->super.executed_tasks++; + remove_gpu_task: + // Load problem: was parsec_device_load[gpu_device->super.device_index] -= gpu_task->load; + parsec_device_load[gpu_device->super.device_index] -= parsec_device_sweight[gpu_device->super.device_index]; + PARSEC_DEBUG_VERBOSE(3, parsec_gpu_output_stream,"GPU[%s]: gpu_task %p freed at %s:%d", gpu_device->super.name, + gpu_task, __FILE__, __LINE__); + free( gpu_task ); + rc = parsec_atomic_fetch_dec_int32( &(gpu_device->mutex) ); + if( 1 == rc ) { /* I was the last one */ +#if defined(PARSEC_PROF_TRACE) + if( parsec_gpu_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_OWN ) + PARSEC_PROFILING_TRACE( es->es_profile, parsec_gpu_own_GPU_key_end, + (unsigned long)es, PROFILE_OBJECT_ID_NULL, NULL ); +#endif /* defined(PARSEC_PROF_TRACE) */ + PARSEC_DEBUG_VERBOSE(2, parsec_gpu_output_stream,"GPU[%s]: Leaving GPU management at %s:%d", + gpu_device->super.name, __FILE__, __LINE__); + + return PARSEC_HOOK_RETURN_ASYNC; + } + gpu_task = progress_task; + goto fetch_task_from_shared_queue; + + disable_gpu: + /* Something wrong happened. Push all the pending tasks back on the + * cores, and disable the gpu. + */ + parsec_warning("Critical issue related to the GPU discovered. Giving up\n"); + return PARSEC_HOOK_RETURN_DISABLE; +} + +#endif /* PARSEC_HAVE_LEVEL_ZERO */ diff --git a/parsec/mca/device/parsec_gpu.c b/parsec/mca/device/parsec_gpu.c new file mode 100644 index 000000000..84fcb3c25 --- /dev/null +++ b/parsec/mca/device/parsec_gpu.c @@ -0,0 +1,97 @@ +/* + * + * Copyright (c) 2021 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + */ + +#include "parsec/parsec_config.h" +#include "parsec/mca/device/device.h" +#include "parsec/mca/device/device_gpu.h" +#include "parsec/utils/mca_param.h" +#include "parsec/mca/mca_repository.h" +#include "parsec/constants.h" +#include "parsec/utils/debug.h" +#include "parsec/execution_stream.h" +#include "parsec/utils/argv.h" +#include "parsec/parsec_internal.h" + +void* parsec_gpu_pop_workspace(parsec_device_gpu_module_t* gpu_device, + parsec_gpu_exec_stream_t* gpu_stream, size_t size) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + (void)gpu_device; (void)gpu_stream; (void)size; + void *work = NULL; + +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + if (gpu_stream->workspace == NULL) { + gpu_stream->workspace = (parsec_gpu_workspace_t *)malloc(sizeof(parsec_gpu_workspace_t)); + gpu_stream->workspace->total_workspace = PARSEC_GPU_MAX_WORKSPACE; + gpu_stream->workspace->stack_head = PARSEC_GPU_MAX_WORKSPACE - 1; + + for( int i = 0; i < PARSEC_GPU_MAX_WORKSPACE; i++ ) { + gpu_stream->workspace->workspace[i] = zone_malloc( gpu_device->memory, size); + PARSEC_DEBUG_VERBOSE(2, parsec_level_zero_output_stream, + "GPU[%s] Succeeded Allocating workspace %d (device_ptr %p)", + gpu_device->super.name, + i, gpu_stream->workspace->workspace[i]); +#if defined(PARSEC_PROF_TRACE) + if((parsec_level_zero_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (level_zero_device->exec_stream[0].super.prof_event_track_enable || + level_zero_device->exec_stream[1].super.prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_level_zero_allocate_memory_key, (int64_t)gpu_stream->workspace->workspace[i], level_zero_device->level_zero_index, + &size, PARSEC_PROFILING_EVENT_COUNTER|PARSEC_PROFILING_EVENT_HAS_INFO); + } +#endif + } + } + assert (gpu_stream->workspace->stack_head >= 0); + work = gpu_stream->workspace->workspace[gpu_stream->workspace->stack_head]; + gpu_stream->workspace->stack_head --; +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return work; +} + +int parsec_gpu_push_workspace(parsec_device_gpu_module_t* gpu_device, parsec_gpu_exec_stream_t* gpu_stream) +{ + (void)gpu_device; (void)gpu_stream; +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + gpu_stream->workspace->stack_head ++; + assert (gpu_stream->workspace->stack_head < PARSEC_GPU_MAX_WORKSPACE); +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return 0; +} + +int parsec_gpu_free_workspace(parsec_device_gpu_module_t * gpu_device) +{ + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; + (void)gpu_device; +#if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) + int i, j; + for( i = 0; i < gpu_device->max_exec_streams; i++ ) { + parsec_gpu_exec_stream_t *gpu_stream = &(level_zero_device->exec_stream[i].super); + if (gpu_stream->workspace != NULL) { + for (j = 0; j < gpu_stream->workspace->total_workspace; j++) { +#if defined(PARSEC_PROF_TRACE) + if((parsec_level_zero_trackable_events & PARSEC_PROFILE_LEVEL_ZERO_TRACK_MEM_USE) && + (level_zero_device->exec_stream[0].super.prof_event_track_enable || + level_zero_device->exec_stream[1].super.prof_event_track_enable)) { + parsec_profiling_trace_flags(gpu_stream->profiling, + parsec_level_zero_allocate_memory_key, (int64_t)gpu_stream->workspace->workspace[i], level_zero_device->level_zero_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + } +#endif + PARSEC_DEBUG_VERBOSE(2, parsec_level_zero_output_stream, + "GPU[%s] Release workspace %d (device_ptr %p)", + gpu_device->super.name, + j, gpu_stream->workspace->workspace[j]); + zone_free( gpu_device->memory, gpu_stream->workspace->workspace[j] ); + } + free(gpu_stream->workspace); + gpu_stream->workspace = NULL; + } + } +#endif /* !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) */ + return 0; +} diff --git a/parsec/mca/device/transfer_gpu.c b/parsec/mca/device/transfer_gpu.c index 1ad93fa9b..439a84724 100644 --- a/parsec/mca/device/transfer_gpu.c +++ b/parsec/mca/device/transfer_gpu.c @@ -132,6 +132,11 @@ static const __parsec_chore_t __gpu_d2h_task_chores[] = { {.type = PARSEC_DEV_CUDA, .evaluate = NULL, .hook = (parsec_hook_t *) hook_of_gpu_d2h_task}, +#endif +#if defined(PARSEC_HAVE_LEVEL_ZERO) + {.type = PARSEC_DEV_LEVEL_ZERO, + .evaluate = NULL, + .hook = (parsec_hook_t *) hook_of_GPU_d2h_task}, #endif {.type = PARSEC_DEV_NONE, .evaluate = NULL, From 7570ac47eb55263f040e4603ad1dd3edf79684af Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Mon, 21 Jun 2021 18:16:35 -0400 Subject: [PATCH 02/15] Add DPC++ to the loop... - Add multiple CMake logic files and commands - jdf2c.c now generates dpcpp output files when needed - make DEV_DPCPP be an alias to DEV_LEVEL_ZERO - Command Lists for I/O (streams of id 0 and 1) are still immediate - Command Lists for computations (streams of id >= 2) are now normal lists connected to a queue that queue exists as a compute level-zero queue and as a DPC++ queue - Missing compilation logic to compile generated dpc++ code and link it with the target binary Risk: it is unclear that the user can still push orders / events in the command list, after it is closed, and it is necessary to close it to force the orders to be pushed on the queue. I might need to create a new command list after each close, and attach the command list to the event for garbage collection. Adapt findlevel-zero.cmake to support systems where pkg-config is broken --- CMakeLists.txt | 8 +- cmake_modules/FindDPCPP.cmake | 44 ++ cmake_modules/Findlevel-zero.cmake | 61 +- parsec/include/parsec/parsec_options.h.in | 1 + parsec/interfaces/ptg/ptg-compiler/jdf.h | 10 +- parsec/interfaces/ptg/ptg-compiler/jdf2c.c | 533 +++++++++++++++++- parsec/interfaces/ptg/ptg-compiler/jdf2c.h | 3 +- parsec/interfaces/ptg/ptg-compiler/main.c | 28 +- parsec/interfaces/ptg/ptg-compiler/parsec.l | 8 +- parsec/interfaces/ptg/ptg-compiler/parsec.y | 30 +- parsec/mca/device/device.h | 1 + parsec/mca/device/device_gpu.c | 4 +- parsec/mca/device/device_gpu.h | 3 +- .../device/level_zero/ValidateModule.CMake | 30 +- .../mca/device/level_zero/device_level_zero.h | 58 +- .../level_zero/device_level_zero_component.c | 12 +- .../level_zero/device_level_zero_dpcpp.h | 27 + .../device_level_zero_dpcpp_interface.cpp | 28 + .../level_zero/device_level_zero_internal.h | 1 + .../level_zero/device_level_zero_module.c | 107 ++-- tests/dsl/ptg/cuda/nvlink.jdf | 20 + 21 files changed, 900 insertions(+), 117 deletions(-) create mode 100644 cmake_modules/FindDPCPP.cmake create mode 100644 parsec/mca/device/level_zero/device_level_zero_dpcpp.h create mode 100644 parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ac7bc8ffc..c8fb925b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -717,12 +717,16 @@ int main(int argc, char *argv[]) { if( PARSEC_GPU_WITH_LEVEL_ZERO ) find_package(level-zero) + find_package(DPCPP) set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero") - if (LEVEL_ZERO_FOUND) + if (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/") set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero") message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}") - endif (LEVEL_ZERO_FOUND) + message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}") + else(LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + message(STATUS "Could not find both level-zero and DPCPP: level-zero says ${LEVEL_ZERO_FOUND}, DPCPP says ${DPCPP_EXECUTABLE}") + endif (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) endif( PARSEC_GPU_WITH_LEVEL_ZERO ) find_package(AYUDAME QUIET) diff --git a/cmake_modules/FindDPCPP.cmake b/cmake_modules/FindDPCPP.cmake new file mode 100644 index 000000000..c4073d268 --- /dev/null +++ b/cmake_modules/FindDPCPP.cmake @@ -0,0 +1,44 @@ +if(DPCPP_EXECUTABLE) + get_filename_component( _dpcpp_path ${DPCPP_EXECUTABLE} PATH ) +else(DPCPP_EXECUTABLE) + set(_dpcpp_path "") +endif(DPCPP_EXECUTABLE) +find_program( DPCPP_EXECUTABLE NAMES dpcpp HINTS ${_dpcpp_path} ENV PATH ) +mark_as_advanced( DPCPP_EXECUTABLE ) + +if( DPCPP_EXECUTABLE ) + execute_process(COMMAND ${DPCPP_EXECUTABLE} --version + RESULT_VARIABLE _res + OUTPUT_VARIABLE _out + ERROR_VARIABLE _err) + if( _res EQUAL 0 ) + string(REGEX MATCH "([^\n]+)" _ ${_out}) + message(STATUS "Found dpcpp: ${DPCPP_EXECUTABLE} version ${CMAKE_MATCH_1}") + + get_filename_component( _dpcpp_dir ${DPCPP_EXECUTABLE} DIRECTORY ) + get_filename_component( _dpcpp_lib_dir "${_dpcpp_dir}/../lib" ABSOLUTE ) + + find_library(SYCL_LIBRARY sycl HINTS "${_dpcpp_lib_dir}" "${SYCL_LIBRARY_DIR}" ENV "SYCL_LIBRARY_DIR") + if( SYCL_LIBRARY ) + message(STATUS "SYCL library: ${SYCL_LIBRARY}") + + get_filename_component( _dpcpp_inc_dir "${_dpcpp_dir}/../include" ABSOLUTE ) + find_file(_sycl_include_file "sycl.hpp" HINTS "${_dpcpp_inc_dir}" "${_dpcpp_inc_dir}/sycl/CL" "${SYCL_INCLUDE_DIR}" ENV "SYCL_INCLUDE_DIR") + if(_sycl_include_file) + get_filename_component(SYCL_INCLUDE_DIR "${_sycl_include_file}" DIRECTORY) + message(STATUS "SYCL include dir: ${SYCL_INCLUDE_DIR}") + set(PARSEC_HAVE_DPCPP "TRUE" CACHE BOOL "True if PaRSEC provide support for dpc++") + else(_sycl_include_file) + message(WARNING "Found DPC++, and SYCL library, but could not find SYCL include directory. Define SYCL_INCLUDE_DIR to enable DPC++ support") + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + endif(_sycl_include_file) + else( SYCL_LIBRARY ) + message(WARNING "Found DPC++, but did not find SYCL library. Define SYCL_LIBRARY_DIR to enable DPC++ support") + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + endif( SYCL_LIBRARY ) + else( _res EQUAL 0 ) + set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++") + message(WARNING "${DPCPP_EXECUTABLE} does not work: 'dpcpp -v' returned ${_res}, with error ${_err}") + message(WARNING "dpc++ support is disabled, set DPCPP_EXECUTABLE to the path of dpcpp to enable DPC++ support, and/or fix your environment to run dpcpp") + endif( _res EQUAL 0 ) +endif( DPCPP_EXECUTABLE ) \ No newline at end of file diff --git a/cmake_modules/Findlevel-zero.cmake b/cmake_modules/Findlevel-zero.cmake index 25ed408b7..4694dcb42 100644 --- a/cmake_modules/Findlevel-zero.cmake +++ b/cmake_modules/Findlevel-zero.cmake @@ -1,15 +1,46 @@ -find_package(PkgConfig QUIET) - -if(PKG_CONFIG_FOUND) - pkg_check_modules(LEVEL_ZERO level-zero) - if(LEVEL_ZERO_FOUND) - pkg_get_variable(LEVEL_ZERO_LIBRARY_DIR level-zero libdir) - pkg_get_variable(LEVEL_ZERO_INCLUDE_DIR level-zero includedir) - add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) - set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so") - set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level-zero/ze_api.h") - include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level-zero/") - endif(LEVEL_ZERO_FOUND) -else(PKG_CONFIG_FOUND) - -endif(PKG_CONFIG_FOUND) \ No newline at end of file +if(LEVEL_ZERO_ROOT_DIR) + message(STATUS "Trying to locate level-zero library and headers under ${LEVEL_ZERO_ROOT_DIR}") + find_library(ZE_LOADER_LIBRARY "ze_loader" HINTS "${LEVEL_ZERO_ROOT_DIR}/lib" "${LEVEL_ZERO_ROOT_DIR}/lib64" NO_DEFAULT_PATH) + find_path (LEVEL_ZERO_INCLUDE_DIR NAMES "level_zero/ze_api.h" PATHS "${LEVEL_ZERO_ROOT_DIR}/include" NO_DEFAULT_PATH) + + if(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) + get_filename_component(ZE_LOADER_LIBRARY_DIR ${ZE_LOADER_LIBRARY} DIRECTORY) + check_library_exists("ze_loader" "zeInit" ${ZE_LOADER_LIBRARY_DIR} ZE_LOADER_HAVE_ZEINIT) + + if(ZE_LOADER_HAVE_ZEINIT) + message(STATUS "Found ze_loader library in ${ZE_LOADER_LIBRARY} and level_zero/ze_api.h in ${LEVEL_ZERO_INCLUDE_DIR}") + add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) + set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${ZE_LOADER_LIBRARY}") + set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h") + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") + set(LEVEL_ZERO_FOUND TRUE) + else(ZE_LOADER_HAVE_ZEINIT) + if(NOT ZE_LOADER_HAVE_ZEINIT) + message(WARNING "Found ze_loader library under ${ZE_LOADER_LIBRARY}, but could not find symbol zeInit in this library -- falling back to package config search") + endif(NOT ZE_LOADER_HAVE_ZEINIT) + endif(ZE_LOADER_HAVE_ZEINIT) + else(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) + if(NOT ZE_LOADER_LIBRARY) + message(WARNING "Could not find ze_loader library under provided LEVEL_ZERO_ROOT_DIR='${LEVEL_ZERO_ROOT_DIR}' (tried subdirectories lib/ and lib64/) -- falling back to package config search") + endif(NOT ZE_LOADER_LIBRARY) + if(NOT LEVEL_ZERO_INCLUDE_DIR) + message(WARNING "Cound not find level_zero/ze_api.h under provided LEVEL_ZERO_ROOT_DIR=${LEVEL_ZERO_ROOT_DIR}' (tried subdirectory include) -- falling back to package config search") + endif(NOT LEVEL_ZERO_INCLUDE_DIR) + endif(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) +endif(LEVEL_ZERO_ROOT_DIR) + +if(NOT LEVEL_ZERO_FOUND) + find_package(PkgConfig QUIET) + + if(PKG_CONFIG_FOUND) + pkg_check_modules(LEVEL_ZERO level-zero) + if(LEVEL_ZERO_FOUND) + pkg_get_variable(LEVEL_ZERO_LIBRARY_DIR level-zero libdir) + pkg_get_variable(LEVEL_ZERO_INCLUDE_DIR level-zero includedir) + add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) + set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so") + set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h") + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") + endif(LEVEL_ZERO_FOUND) + endif(PKG_CONFIG_FOUND) +endif(NOT LEVEL_ZERO_FOUND) diff --git a/parsec/include/parsec/parsec_options.h.in b/parsec/include/parsec/parsec_options.h.in index 3bfd8b0cd..15635a03c 100644 --- a/parsec/include/parsec/parsec_options.h.in +++ b/parsec/include/parsec/parsec_options.h.in @@ -122,6 +122,7 @@ #cmakedefine PARSEC_HAVE_PAPI #cmakedefine PARSEC_HAVE_CUDA #cmakedefine PARSEC_HAVE_LEVEL_ZERO +#cmakedefine PARSEC_HAVE_DPCPP #cmakedefine PARSEC_HAVE_OPENCL #cmakedefine PARSEC_HAVE_MPI #cmakedefine PARSEC_HAVE_MPI_20 diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf.h b/parsec/interfaces/ptg/ptg-compiler/jdf.h index 189d276f7..0d9f268ce 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf.h +++ b/parsec/interfaces/ptg/ptg-compiler/jdf.h @@ -96,6 +96,7 @@ typedef struct jdf_compiler_global_args { char *input; char *output_c; char *output_h; + char *output_driver_basename; char *output_o; char *funcid; jdf_warning_mask_t wmask; @@ -139,10 +140,17 @@ extern int jdfdebug; * We remember the line number in the JDF file where this external code was found */ typedef struct jdf_external_entry { - struct jdf_object_t super; + struct jdf_object_t super; + struct jdf_external_entry *next; + char *language; char *external_code; } jdf_external_entry_t; +typedef struct jdf_code_string { + char *language; + char *string; +} jdf_code_string_t; + /** A global is a variable name, optionally an expression to define it, * and a line number associated with it for error printing purposes */ diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index ee7d0060a..72342d266 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -3,7 +3,6 @@ * of Tennessee Research Foundation. All rights * reserved. */ - #include "parsec/parsec_config.h" #include @@ -31,10 +30,14 @@ extern const char *yyfilename; static FILE *cfile; static int cfile_lineno; +static FILE *dpcppfile = NULL; +static int dpcppfile_lineno = 0; static FILE *hfile; static int hfile_lineno; static const char *jdf_basename; static const char *jdf_cfilename; +static char *jdf_hfilename; +static char *jdf_dpcppfilename; /* Optional declarations of local functions */ static int jdf_expr_depends_on_symbol(const char *varname, const jdf_expr_t *expr); @@ -133,6 +136,43 @@ static void coutput(const char *format, ...) } } +#if defined(__GNUC__) +static void dpcpp_output(const char *format, ...) __attribute__((format(printf,1,2))); +#endif +static void dpcpp_output(const char *format, ...) +{ + va_list ap; + char *res; + int len; + + va_start(ap, format); + len = vasprintf(&res, format, ap); + va_end(ap); + + if( len == -1 ) { + fprintf(stderr, "Unable to ouptut a string: %s\n", strerror(errno)); + } else if( 0 < len ) { +#if (defined(__WINDOWS__) || defined(__CYGWIN__)) && !defined(__MINGW64__) + char *start = res, *end; + while( NULL != (end = strchr(start, '\n'))) { + if( (end != start) && (end[-1] != '\r')) { + fwrite(start, (end - start), 1, dpcppfile); + fwrite("\r\n", 2, 1, dpcppfile); + } else { + fwrite(start, (end - start) + 1, 1, dpcppfile); + } + len -= (end - start) + 1; + start = end + 1; /* skip the current \n */ + } + fwrite(start, len, 1, dpcppfile); +#else + fwrite(res, len, 1, dpcppfile); +#endif /* (defined(__WINDOWS__) || defined(__CYGWIN__)) && !defined(__MINGW64__) */ + dpcppfile_lineno += nblines(res); + free(res); + } +} + #if defined(__GNUC__) static void houtput(const char *format, ...) __attribute__((format(printf,1,2))); #endif @@ -1499,6 +1539,9 @@ static void jdf_minimal_code_before_prologue(const jdf_t *jdf) "#if defined(PARSEC_HAVE_CUDA)\n" "#include \"parsec/mca/device/cuda/device_cuda.h\"\n" "#endif /* defined(PARSEC_HAVE_CUDA) */\n" + "#if defined(PARSEC_HAVE_LEVEL_ZERO)\n" + "#include \"parsec/mca/device/level_zero/device_level_zero.h\"\n" + "#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */\n" "#if defined(_MSC_VER) || defined(__MINGW32__)\n" "# include \n" "#else\n" @@ -1515,9 +1558,9 @@ static void jdf_minimal_code_before_prologue(const jdf_t *jdf) jdf_generate_predeclarations(jdf); } -static void jdf_generate_structure(jdf_t *jdf) +static void jdf_dump_internal_structure(string_arena_t *sa, jdf_t *jdf) { - int nbfunctions, need_profile = 0; + int nbfunctions = 0, need_profile = 0; string_arena_t *sa1, *sa2; jdf_function_entry_t* f; jdf_param_list_t *pl; @@ -1527,7 +1570,7 @@ static void jdf_generate_structure(jdf_t *jdf) sa1 = string_arena_new(64); sa2 = string_arena_new(64); - coutput("#include \"%s.h\"\n\n" + string_arena_add_string(sa, "#include \"%s.h\"\n\n" "struct __parsec_%s_internal_taskpool_s {\n" " parsec_%s_taskpool_t super;\n" " volatile int32_t sync_point;\n" @@ -1535,30 +1578,46 @@ static void jdf_generate_structure(jdf_t *jdf) " parsec_task_t* startup_queue;\n", jdf_basename, jdf_basename, jdf_basename); - coutput(" /* The ranges to compute the hash key */\n"); + string_arena_add_string(sa, " /* The ranges to compute the hash key */\n"); for(f = jdf->functions; f != NULL; f = f->next) { if( 0 == (f->user_defines & JDF_FUNCTION_HAS_UD_MAKE_KEY) ) { for(pl = f->parameters; pl != NULL; pl = pl->next) { - coutput(" int %s_%s_range;\n", f->fname, pl->name); + string_arena_add_string(sa, " int %s_%s_range;\n", f->fname, pl->name); } } else { - coutput(" /* nothing for %s as it gets a user-defined make_key */\n", + string_arena_add_string(sa, " /* nothing for %s as it gets a user-defined make_key */\n", f->fname); } } - coutput(" /* The list of data repositories "); + string_arena_add_string(sa, " /* The list of data repositories "); for(f = jdf->functions; NULL != f; f = f->next) { if( 0 != function_has_data_output(f) ) { - coutput(" %s ", f->fname); + string_arena_add_string(sa, " %s ", f->fname); } } - coutput("*/\n"); + string_arena_add_string(sa, "*/\n"); if(nbfunctions != 0 ) { - coutput(" data_repo_t* repositories[%d];\n", nbfunctions ); + string_arena_add_string(sa, " data_repo_t* repositories[%d];\n", nbfunctions ); } - coutput("};\n\n"); + string_arena_add_string(sa, "};\n\n"); +} + +static void jdf_generate_structure(jdf_t *jdf) +{ + int nbfunctions, need_profile = 0; + string_arena_t *sa1, *sa2; + jdf_function_entry_t* f; + + JDF_COUNT_LIST_ENTRIES(jdf->functions, jdf_function_entry_t, next, nbfunctions); + + sa1 = string_arena_new(64); + sa2 = string_arena_new(64); + + jdf_dump_internal_structure(sa1, jdf); + coutput("%s\n", string_arena_get_string(sa1)); + string_arena_init(sa1); for( f = jdf->functions; need_profile == 0 && NULL != f; f = f->next ) { /* If the profile property is ON then enable the profiling array */ @@ -6554,6 +6613,25 @@ static int jdf_has_cuda_chore(const jdf_t *jdf, const char *fname) return 0; } +static int jdf_has_dpcpp_chore(const jdf_t *jdf, const char *fname) +{ + jdf_function_entry_t *f; + jdf_body_t* body; + jdf_def_list_t *type_property; + + for(f = jdf->functions; f != NULL; f = f->next) { + if( (NULL != fname) && strcmp(f->fname, fname) ) continue; + for(body = f->bodies; body != NULL; body = body->next) { + jdf_find_property(body->properties, "type", &type_property); + if( NULL != type_property && !strcmp(type_property->expr->jdf_var, "DPCPP")) + return 1; + } + if(NULL != fname) return 0; + } + return 0; +} + + static void jdf_generate_code_hook_cuda(const jdf_t *jdf, const jdf_function_entry_t *f, const jdf_body_t* body, @@ -6901,6 +6979,335 @@ static void jdf_generate_code_hook_cuda(const jdf_t *jdf, string_arena_free(sa3); } +static void jdf_generate_code_hook_dpcpp(const jdf_t *jdf, + const jdf_function_entry_t *f, + const jdf_body_t* body, + const char *name) +{ + jdf_def_list_t *type_property; + jdf_def_list_t *stage_in_property; + jdf_def_list_t *stage_out_property; + jdf_def_list_t *size_property; + jdf_def_list_t *desc_property; + jdf_def_list_t *weight_property; + jdf_def_list_t *device_property; + const char *device; + const char *weight; + string_arena_t *sa, *sa2, *sa3; + assignment_info_t ai; + init_from_data_info_t ai2; + jdf_dataflow_t *fl; + expr_info_t info = EMPTY_EXPR_INFO; + int di; + int profile_on; + char* output; + + profile_on = profile_enabled(f->properties) && profile_enabled(body->properties); + + jdf_find_property(body->properties, "type", &type_property); + + sa = string_arena_new(64); + sa2 = string_arena_new(64); + sa3 = string_arena_new(64); + + ai.sa = sa2; + ai.holder = "this_task->locals."; + ai.expr = NULL; + + string_arena_add_string(sa3, "%s", + UTIL_DUMP_LIST(sa, f->locals, next, + dump_local_assignments, &ai, "", " ", "\n", "\n")); + + string_arena_add_string(sa3, "%s", + UTIL_DUMP_LIST_FIELD(sa, f->locals, next, name, + dump_string, NULL, "", " (void)", ";", ";\n")); + + /* Generate the dpcpp_kernel_submit structure and function */ + dpcpp_output("extern \"C\" {\n" + " int parsec_dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream);\n" + "}\n" + "\n" + "int parsec_dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream )\n" + "{\n" + " %s *this_task = reinterpret_cast<%s *>(gpu_task->ec);\n" + " parsec_device_level_zero_module_t *level_zero_device = reinterpret_cast(gpu_device);\n" + " parsec_level_zero_exec_stream_t *level_zero_stream = reinterpret_cast(gpu_stream);\n" + " __parsec_%s_internal_taskpool_t *__parsec_tp = reinterpret_cast<__parsec_%s_internal_taskpool_t *>(this_task->taskpool);\n" + " parsec_dpcpp_object_t *parsec_dpcpp = reinterpret_cast(level_zero_stream->dpcpp_obj);\n" + "\n" + "%s\n" + " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)level_zero_device; (void)level_zero_stream;\n", + jdf_basename, f->fname, + jdf_basename, f->fname, + parsec_get_name(jdf, f, "task_t"), parsec_get_name(jdf, f, "task_t"), + jdf_basename, jdf_basename, + string_arena_get_string( sa3 )); + + ai2.sa = sa2; + ai2.where = "out"; + output = UTIL_DUMP_LIST(sa, f->dataflow, next, + dump_data_initialization_from_data_array, &ai2, "", "", "", ""); + if( 0 != strlen(output) ) { + dpcpp_output(" /** Declare the variables that will hold the data, and all the accounting for each */\n" + "%s\n", + output); + } + + /** + * Generate code for the simulation. + */ + dpcpp_output(" /** Update starting simulation date */\n" + "#if defined(PARSEC_SIM)\n" + " {\n" + " this_task->sim_exec_date = 0;\n"); + for( di = 0, fl = f->dataflow; fl != NULL; fl = fl->next, di++ ) { + + if(fl->flow_flags & JDF_FLOW_TYPE_CTL) continue; /* control flow, nothing to store */ + + dpcpp_output(" data_repo_entry_t *e%s = this_task->data._f_%s.source_repo_entry;\n" + " if( (NULL != e%s) && (e%s->sim_exec_date > this_task->sim_exec_date) )\n" + " this_task->sim_exec_date = e%s->sim_exec_date;\n", + fl->varname, fl->varname, + fl->varname, fl->varname, + fl->varname); + } + dpcpp_output(" if( this_task->task_class->sim_cost_fct != NULL ) {\n" + " this_task->sim_exec_date += this_task->task_class->sim_cost_fct(this_task);\n" + " }\n" + " if( es->largest_simulation_date < this_task->sim_exec_date )\n" + " es->largest_simulation_date = this_task->sim_exec_date;\n" + " }\n" + "#endif\n"); + + //TODO: jdf_generate_code_cache_awareness_update(jdf, f); + + dpcpp_output("#if defined(PARSEC_DEBUG_NOISIER)\n" + " {\n" + " char tmp[MAX_TASK_STRLEN];\n" + " PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, \"GPU[%%s]:\\tEnqueue on device %%s priority %%d\", gpu_device->super.name, \n" + " parsec_task_snprintf(tmp, MAX_TASK_STRLEN, (parsec_task_t *)this_task),\n" + " this_task->priority );\n" + " }\n" + "#endif /* defined(PARSEC_DEBUG_NOISIER) */\n" ); + + if( profile_on ) { + dpcpp_output("#if defined(PARSEC_PROF_TRACE)\n" + " if(gpu_stream->prof_event_track_enable) {\n" + " PARSEC_TASK_PROF_TRACE(gpu_stream->profiling,\n" + " PARSEC_PROF_FUNC_KEY_START(this_task->taskpool,\n" + " this_task->task_class->task_class_id),\n" + " (parsec_task_t*)this_task);\n" + " gpu_task->prof_key_end = PARSEC_PROF_FUNC_KEY_END(this_task->taskpool,\n" + " this_task->task_class->task_class_id);\n" + " gpu_task->prof_event_id = this_task->task_class->key_functions->\n" + " key_hash(this_task->task_class->make_key(this_task->taskpool, ((parsec_task_t*)this_task)->locals), NULL);\n" + " gpu_task->prof_tp_id = this_task->taskpool->taskpool_id;\n" + " }\n" + "#endif /* PARSEC_PROF_TRACE */\n"); + } + + dpcpp_output("%s\n", body->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) { + dpcpp_output("#line %d \"%s\"\n", dpcppfile_lineno+1, jdf_dpcppfilename); + } + + // TODO: jdf_generate_code_dry_run_after(jdf, f); + dpcpp_output(" return PARSEC_HOOK_RETURN_DONE;\n" + "}\n\n"); + + /* Generate the hook_level_zero */ + coutput("extern int parsec_dpcpp_kernel_submit_%s_%s(parsec_device_gpu_module_t *gpu_device,\n" + " parsec_gpu_task_t *gpu_task,\n" + " parsec_gpu_exec_stream_t *gpu_stream );\n" + "\n" + "static int %s_%s(parsec_execution_stream_t *es, %s *this_task)\n" + "{\n" + " __parsec_%s_internal_taskpool_t *__parsec_tp = (__parsec_%s_internal_taskpool_t *)this_task->taskpool;\n" + " parsec_gpu_task_t *gpu_task;\n" + " double ratio;\n" + " int dev_index;\n" + " %s\n" + " (void)es; (void)__parsec_tp;\n" + "\n", + jdf_basename, f->fname, + name, type_property->expr->jdf_var, parsec_get_name(jdf, f, "task_t"), + jdf_basename, jdf_basename, + string_arena_get_string( sa3 )); + + info.sa = string_arena_new(64); + info.prefix = ""; + info.suffix = ""; + info.assignments = "&this_task->locals"; + + /* Get the ratio to apply on the weight for this task */ + jdf_find_property( body->properties, "weight", &weight_property ); + if ( NULL != weight_property ) { + weight = dump_expr((void**)weight_property->expr, &info); + } else { + weight = "1."; + } + coutput(" ratio = %s;\n", weight); + + /* Get the hint for statix and/or external gpu scheduling */ + jdf_find_property( body->properties, "device", &device_property ); + if ( NULL != device_property ) { + device = dump_expr((void**)device_property->expr, &info); + coutput(" dev_index = %s;\n" + " if (dev_index < -1) {\n" + " return PARSEC_HOOK_RETURN_NEXT;\n" + " } else if (dev_index == -1) {\n" + " dev_index = parsec_gpu_get_best_device((parsec_task_t*)this_task, ratio);\n" + " } else {\n" + " dev_index = (dev_index %% (parsec_mca_device_enabled()-2)) + 2;\n" + " }\n", + device); + } else { + coutput(" dev_index = parsec_gpu_get_best_device((parsec_task_t*)this_task, ratio);\n"); + } + coutput(" assert(dev_index >= 0);\n" + " if( dev_index < 2 ) {\n" + " return PARSEC_HOOK_RETURN_NEXT; /* Fall back */\n" + " }\n" + "\n" + " gpu_task = (parsec_gpu_task_t*)calloc(1, sizeof(parsec_gpu_task_t));\n" + " PARSEC_OBJ_CONSTRUCT(gpu_task, parsec_list_item_t);\n" + " gpu_task->ec = (parsec_task_t*)this_task;\n" + " gpu_task->submit = &parsec_dpcpp_kernel_submit_%s_%s;\n" + " gpu_task->task_type = 0;\n" + " gpu_task->load = ratio * parsec_device_sweight[dev_index];\n" + " gpu_task->last_data_check_epoch = -1; /* force at least one validation for the task */\n", + jdf_basename, f->fname); + + /* Set up stage in/out callbacks */ + jdf_find_property(body->properties, "stage_in", &stage_in_property); + jdf_find_property(body->properties, "stage_out", &stage_out_property); + + if(stage_in_property == NULL) { + coutput(" gpu_task->stage_in = parsec_default_level_zero_stage_in;\n"); + }else{ + coutput(" gpu_task->stage_in = %s;\n", dump_expr((void**)stage_in_property->expr, &info)); + } + + if(stage_out_property == NULL) { + coutput(" gpu_task->stage_out = parsec_default_level_zero_stage_out;\n"); + }else{ + coutput(" gpu_task->stage_out = %s;\n", dump_expr((void**)stage_out_property->expr, &info)); + } + + /* Dump the dataflow */ + coutput(" gpu_task->pushout = 0;\n"); + for(fl = f->dataflow, di = 0; fl != NULL; fl = fl->next, di++) { + coutput(" gpu_task->flow[%d] = &%s;\n", + di, JDF_OBJECT_ONAME( fl )); + + sprintf(sa->ptr, "%s.dc", fl->varname); + jdf_find_property(body->properties, sa->ptr, &desc_property); + if(desc_property == NULL){ + coutput(" gpu_task->flow_dc[%d] = NULL;\n", di); + }else{ + coutput(" gpu_task->flow_dc[%d] = (parsec_data_collection_t *)%s;\n", di, + dump_expr((void**)desc_property->expr, &info)); + } + + sprintf(sa->ptr, "%s.size", fl->varname); + jdf_find_property(body->properties, sa->ptr, &size_property); + + if(fl->flow_flags & JDF_FLOW_TYPE_CTL) { + if(size_property != NULL){ + fprintf(stderr, "Error: specifying GPU buffer size for CTL flow %s at line %d\n", + fl->varname, JDF_OBJECT_LINENO(fl)); + exit(-1); + } + coutput(" gpu_task->flow_nb_elts[%d] = 0;\n", di); + }else{ + if(size_property == NULL){ + coutput(" gpu_task->flow_nb_elts[%d] = gpu_task->ec->data[%d].data_in->original->nb_elts;\n", di, di); + }else{ + coutput(" gpu_task->flow_nb_elts[%d] = %s;\n", + di, dump_expr((void**)size_property->expr, &info)); + if( (stage_in_property == NULL) || ( stage_out_property == NULL )){ + coutput(" assert(gpu_task->ec->data[%d].data_in->original->nb_elts <= %s);\n", + di, dump_expr((void**)size_property->expr, &info)); + } + + } + } + + if (fl->flow_flags & JDF_FLOW_TYPE_WRITE) { + jdf_dep_t *dl; + int testtrue, testfalse; + + /** + * We force the pushout for every data that is not only going to the + * same kind of kernel in the future. + * (TODO: could be avoided with different GPU compliant kernels) + */ + for(dl = fl->deps; dl != NULL; dl = dl->next) { + if( dl->dep_flags & JDF_DEP_FLOW_IN ) + continue; + + testtrue = (dl->guard->calltrue != NULL) && + ((dl->guard->calltrue->var == NULL ) || + (!jdf_has_dpcpp_chore(jdf, dl->guard->calltrue->func_or_mem))); + + testfalse = (dl->guard->callfalse != NULL) && + ((dl->guard->callfalse->var == NULL ) || + (!jdf_has_dpcpp_chore(jdf, dl->guard->callfalse->func_or_mem))); + + switch( dl->guard->guard_type ) { + case JDF_GUARD_UNCONDITIONAL: + if(testtrue) { + coutput(" gpu_task->pushout |= (1 << %d);\n", di); + goto nextflow; + } + break; + case JDF_GUARD_BINARY: + if(testtrue) { + coutput(" if( %s ) {\n" + " gpu_task->pushout |= (1 << %d);\n" + " }", + dump_expr((void**)dl->guard->guard, &info), di); + } + break; + case JDF_GUARD_TERNARY: + if( testtrue ) { + if( testfalse ) { + coutput(" gpu_task->pushout |= (1 << %d);\n", di); + } else { + coutput(" if( %s ) {\n" + " gpu_task->pushout |= (1 << %d);\n" + " }\n", + dump_expr((void**)dl->guard->guard, &info), di); + } + } else if ( testfalse ) { + coutput(" if( !(%s) ) {\n" + " gpu_task->pushout |= (1 << %d);\n" + " }\n", + dump_expr((void**)dl->guard->guard, &info), di); + } + break; + } + } + nextflow: + ; + } + } + string_arena_free(info.sa); + + coutput(" parsec_device_load[dev_index] += gpu_task->load;\n" + "\n" + " return parsec_level_zero_kernel_scheduler( es, gpu_task, dev_index );\n" + "}\n\n"); + + string_arena_free(sa); + string_arena_free(sa2); + string_arena_free(sa3); +} + static void jdf_generate_code_hook(const jdf_t *jdf, const jdf_function_entry_t *f, const jdf_body_t* body, @@ -6940,6 +7347,11 @@ static void jdf_generate_code_hook(const jdf_t *jdf, jdf_generate_code_hook_cuda(jdf, f, body, name); goto hook_end_block; } + + if(!strcmp(type_property->expr->jdf_var, "DPCPP")) { + jdf_generate_code_hook_dpcpp(jdf, f, body, name); + goto hook_end_block; + } } sa = string_arena_new(64); sa2 = string_arena_new(64); @@ -7004,7 +7416,7 @@ static void jdf_generate_code_hook(const jdf_t *jdf, if ((NULL == type_property) || (!strcmp(type_property->expr->jdf_var, "RECURSIVE"))) { coutput(" /** Transfer the ownership to the CPU */\n" - "#if defined(PARSEC_HAVE_CUDA)\n"); + "#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_LEVEL_ZERO)\n"); for( di = 0, fl = f->dataflow; fl != NULL; fl = fl->next, di++ ) { /* Update the ownership of read/write data */ @@ -7022,7 +7434,7 @@ static void jdf_generate_code_hook(const jdf_t *jdf, ((fl->flow_flags & JDF_FLOW_TYPE_WRITE) ? "PARSEC_FLOW_ACCESS_RW" : "PARSEC_FLOW_ACCESS_READ") : "PARSEC_FLOW_ACCESS_WRITE"))); } } - coutput("#endif /* defined(PARSEC_HAVE_CUDA) */\n"); + coutput("#endif /* defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_LEVEL_ZERO) */\n"); } jdf_generate_code_cache_awareness_update(jdf, f); @@ -8364,14 +8776,48 @@ int jdf_optimize( jdf_t* jdf ) #include #endif -int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, jdf_t *jdf) +int jdf2c(const char *output_c, const char *output_h, + const char *driver_output_basename, const char *_jdf_basename, jdf_t *jdf) { int ret = 0; jdf_cfilename = output_c; + jdf_hfilename = strdup(output_h); + asprintf(&jdf_dpcppfilename, "%s.dpcpp.C", driver_output_basename); jdf_basename = _jdf_basename; cfile = NULL; hfile = NULL; + dpcppfile = NULL; + + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa1, *sa2; + sa1 = string_arena_new(64); + sa2 = string_arena_new(64); + dpcppfile = fopen(jdf_dpcppfilename, "w"); + if( dpcppfile == NULL ) { + fprintf(stderr, "unable to create %s: %s\n", jdf_dpcppfilename, strerror(errno)); + ret = -1; + goto err; + } + + UTIL_DUMP_LIST(sa1, jdf->globals, next, + dump_globals, sa2, "", "#define ", "\n", "\n"); + + dpcpp_output("#include \"parsec.h\"\n" + "#include \"level_zero/ze_api.h\"\n" + "#include \"sycl/backend/level_zero.hpp\"\n" + "#include \"parsec/parsec_internal.h\"\n" + "#include \"parsec/execution_stream.h\"\n" + "#include \"parsec/mca/device/device.h\"\n" + "#include \"parsec/mca/device/device_gpu.h\"\n" + "#include \"parsec/mca/device/level_zero/device_level_zero.h\"\n" + "#include \"parsec/mca/device/level_zero/device_level_zero_dpcpp.h\"\n" + "\n" + "%s\n", + string_arena_get_string(sa1)); + string_arena_free(sa1); + string_arena_free(sa2); + } #if defined(PARSEC_HAVE_INDENT) && !(defined(__WINDOWS__) || defined(__MING64__) || defined(__CYGWIN__)) /* When we apply indent/awk to the output of jdf2c, we need to make @@ -8459,6 +8905,7 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, cfile_lineno = 1; hfile_lineno = 1; + dpcppfile_lineno = 1; /** * Now generate the code. @@ -8475,10 +8922,30 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, /** * Dump the prologue section */ - if( NULL != jdf->prologue ) { - coutput("%s", jdf->prologue->external_code); - if( !JDF_COMPILER_GLOBAL_ARGS.noline ) - coutput("#line %d \"%s\"\n", cfile_lineno+1, jdf_cfilename); + for( jdf_external_entry_t *ent = jdf->prologue; NULL != ent; ent = ent->next ) { + if( 0 == strcmp(ent->language, "C") ) { + coutput("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + coutput("#line %d \"%s\"\n", cfile_lineno + 1, jdf_cfilename); + } else if( 0 == strcmp(ent->language, "DPCPP") ) { + dpcpp_output("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + dpcpp_output("#line %d \"%s\"\n", dpcppfile_lineno + 1, jdf_dpcppfilename); + } else { + jdf_warn(ent->super.lineno, "Unknown language '%s' in prologue: code ignored", ent->language); + } + } + + /* Now we can include the header generated to define the datatypes */ + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa = string_arena_new(64); + jdf_dump_internal_structure(sa, jdf); + dpcpp_output("%s\n" + "typedef struct __parsec_%s_internal_taskpool_s __parsec_%s_internal_taskpool_t;\n" + "\n", + string_arena_get_string(sa), + jdf_basename, jdf_basename); + string_arena_free(sa); } /* Dump references to arenas_datatypes array */ @@ -8518,10 +8985,18 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, /** * Dump all the epilogue sections */ - if( NULL != jdf->epilogue ) { - coutput("%s", jdf->epilogue->external_code); - if( !JDF_COMPILER_GLOBAL_ARGS.noline ) - coutput("#line %d \"%s\"\n",cfile_lineno+1, jdf_cfilename); + for( jdf_external_entry_t *ent = jdf->epilogue; NULL != ent; ent = ent->next ) { + if( 0 == strcmp(ent->language, "C") ) { + coutput("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + coutput("#line %d \"%s\"\n", cfile_lineno + 1, jdf_cfilename); + } else if( 0 == strcmp(ent->language, "DPCPP") ) { + dpcpp_output("%s", ent->external_code); + if( !JDF_COMPILER_GLOBAL_ARGS.noline ) + dpcpp_output("#line %d \"%s\"\n", dpcppfile_lineno + 1, jdf_dpcppfilename); + } else { + jdf_warn(ent->super.lineno, "Unknown language '%s' in epilogue: code ignored", ent->language); + } } /** @@ -8535,10 +9010,22 @@ int jdf2c(const char *output_c, const char *output_h, const char *_jdf_basename, err: if( NULL != cfile ) { fclose(cfile); + cfile = NULL; } if( NULL != hfile ) { fclose(hfile); + hfile = NULL; + } + + if( NULL != dpcppfile ) { + fclose(dpcppfile); + dpcppfile = NULL; + } + + if(NULL != jdf_dpcppfilename) { + free(jdf_dpcppfilename); + jdf_dpcppfilename = NULL; } #if defined(PARSEC_HAVE_INDENT) && !(defined(__WINDOWS__) || defined(__MING64__) || defined(__CYGWIN__)) diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.h b/parsec/interfaces/ptg/ptg-compiler/jdf2c.h index f87ad1c7b..5c36913c8 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.h +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.h @@ -10,6 +10,7 @@ int jdf_optimize( jdf_t* jdf ); -int jdf2c(const char *output_c, const char *output_h, const char *_basename, jdf_t *jdf); +int jdf2c(const char *output_c, const char *output_h, + const char *driver_basename, const char *_basename, jdf_t *jdf); #endif /* _jdf2c_h */ diff --git a/parsec/interfaces/ptg/ptg-compiler/main.c b/parsec/interfaces/ptg/ptg-compiler/main.c index 4ff11c5eb..1826ee850 100644 --- a/parsec/interfaces/ptg/ptg-compiler/main.c +++ b/parsec/interfaces/ptg/ptg-compiler/main.c @@ -29,6 +29,7 @@ static jdf_compiler_global_args_t DEFAULTS = { .input = "-", .output_c = "a.c", .output_h = "a.h", + .output_driver_basename = "a", .output_o = "a.o", .funcid = "a", .wmask = JDF_ALL_WARNINGS, @@ -55,10 +56,13 @@ static void usage(void) " --debug|-d Enable debug output\n" " --input|-i Input File (JDF) (default '%s')\n" " --output|-o Set the BASE name for .c, .h, .o and function name (no default).\n" - " Changing this value has precendence over the defaults of\n" + " Changing this value has precedence over the defaults of\n" " --output-c, --output-h, and --function-name\n" " --output-c|-C Set the name of the .c output file (default '%s' or BASE.c)\n" " --output-h|-H Set the name of the .h output file (default '%s' or BASE.h)\n" + " --output-driver Set the base name for additional driver files (e.g. DPC++ bodies).\n" + " (default '%s' or BASE). Changing this value has precedence over\n" + " the defaults of --output\n" " --function-name|-f Set the unique identifier of the generated function\n" " The generated function will be called PaRSEC__new\n" " (default %s)\n" @@ -89,6 +93,7 @@ static void usage(void) DEFAULTS.input, DEFAULTS.output_c, DEFAULTS.output_h, + DEFAULTS.output_driver_basename, DEFAULTS.funcid, (DEFAULTS.dep_management == DEP_MANAGEMENT_INDEX_ARRAY ? DEP_MANAGEMENT_INDEX_ARRAY_STRING : (DEFAULTS.dep_management == DEP_MANAGEMENT_DYNAMIC_HASH_TABLE ? DEP_MANAGEMENT_DYNAMIC_HASH_TABLE_STRING : @@ -164,6 +169,7 @@ static void parse_args(int argc, char *argv[]) char *O = NULL; char *h = NULL; char *o = NULL; + char *d = NULL; char *f = NULL; struct option longopts[] = { @@ -172,6 +178,7 @@ static void parse_args(int argc, char *argv[]) { "output-c", required_argument, NULL, 'C' }, { "output-h", required_argument, NULL, 'H' }, { "output-o", required_argument, NULL, 'O' }, + { "output-driver", required_argument, NULL, 'D' }, { "output", required_argument, NULL, 'o' }, { "function-name", required_argument, NULL, 'f' }, { "Wmasked", no_argument, &wmasked, 1 }, @@ -196,7 +203,7 @@ static void parse_args(int argc, char *argv[]) print_jdf_line = !DEFAULTS.noline; - while( (ch = getopt_long(argc, argv, "di:C:H:o:f:hEsIO:M:I:", longopts, NULL)) != -1) { + while( (ch = getopt_long(argc, argv, "di:C:H:o:D:f:hEsIO:M:I:", longopts, NULL)) != -1) { switch(ch) { case 'd': yydebug = 1; @@ -227,6 +234,11 @@ static void parse_args(int argc, char *argv[]) free( o ); o = strdup(optarg); break; + case 'D': + if( NULL != d) + free( d ); + d = strdup(optarg); + break; case 'f': if( NULL != f ) free( f ); @@ -306,6 +318,15 @@ static void parse_args(int argc, char *argv[]) JDF_COMPILER_GLOBAL_ARGS.output_o = DEFAULTS.output_o; } + if( NULL != d ) { + JDF_COMPILER_GLOBAL_ARGS.output_driver_basename = d; + } else { + if(NULL != o) { + JDF_COMPILER_GLOBAL_ARGS.output_driver_basename = strdup(o); + } else + JDF_COMPILER_GLOBAL_ARGS.output_driver_basename = DEFAULTS.output_driver_basename; + } + if( NULL == c) { if( NULL != o ) { JDF_COMPILER_GLOBAL_ARGS.output_c = (char*)malloc(strlen(o) + 3); @@ -348,6 +369,8 @@ static void parse_args(int argc, char *argv[]) free(h); if( NULL != o ) free(o); + if( NULL != d ) + free(d); if( print_compile_cmd ) { /* print the compilation options used to compile the preprocessed output */ @@ -413,6 +436,7 @@ int main(int argc, char *argv[]) if( jdf2c(JDF_COMPILER_GLOBAL_ARGS.output_c, JDF_COMPILER_GLOBAL_ARGS.output_h, + JDF_COMPILER_GLOBAL_ARGS.output_driver_basename, JDF_COMPILER_GLOBAL_ARGS.funcid, ¤t_jdf) < 0 ) { return 1; diff --git a/parsec/interfaces/ptg/ptg-compiler/parsec.l b/parsec/interfaces/ptg/ptg-compiler/parsec.l index edb6a4e0c..cbf015a3e 100644 --- a/parsec/interfaces/ptg/ptg-compiler/parsec.l +++ b/parsec/interfaces/ptg/ptg-compiler/parsec.l @@ -110,7 +110,8 @@ extern{WHITE}+\"[^\"]+\"{WHITE}+\%\{{WHITE}*[\r\n]+([^\%]|(\%[^\}]))*\%\} /* find the end of the language */ for(lend = lstart + 1; yytext[lend] != '"'; lend++) /* nothing */; - assert( (yytext[lstart] == 'C') && ((lend-lstart) == 1) ); + YYLVAL(code).language = (char*)calloc(lend-lstart+1, 1); + memcpy(YYLVAL(code).language, &yytext[lstart], lend-lstart); /* Search for the start of the body */ for(bstart = lend + 1; yytext[bstart] != '{'; bstart++ ) /* nothing */; @@ -120,14 +121,15 @@ extern{WHITE}+\"[^\"]+\"{WHITE}+\%\{{WHITE}*[\r\n]+([^\%]|(\%[^\}]))*\%\} for( lend = 0; lend <= bstart; lend++ ) if( yytext[lend] == '\n' ) skip_line++; current_lineno += skip_line; - YYLVAL(string) = extern_code( yytext + bstart, strlen(yytext)-bstart-2, 1 ); + YYLVAL(code).string = extern_code( yytext + bstart, strlen(yytext)-bstart-2, 1 ); ECHO; return EXTERN_DECL; } (inline_c)?{WHITE}*\%c?\{([^\%]|(\%[^\}]))+\%\} { int start; for(start = 0; yytext[start] != '{'; start++) /* nothing */ ; start++; - YYLVAL(string) = extern_code( yytext + start, strlen(yytext)-start-2, 0 ); + YYLVAL(code).string = extern_code( yytext + start, strlen(yytext)-start-2, 0 ); + YYLVAL(code).language = strdup("C"); ECHO; return EXTERN_DECL; } SIMCOST { ECHO; return SIMCOST; } diff --git a/parsec/interfaces/ptg/ptg-compiler/parsec.y b/parsec/interfaces/ptg/ptg-compiler/parsec.y index 92393ca41..0478bdbe8 100644 --- a/parsec/interfaces/ptg/ptg-compiler/parsec.y +++ b/parsec/interfaces/ptg/ptg-compiler/parsec.y @@ -271,6 +271,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, %union { int number; + jdf_code_string_t code; char* string; jdf_expr_operand_t expr_op; jdf_external_entry_t *external_code; @@ -291,7 +292,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, }; %glr-parser -%expect 5 +%expect 6 %type function %type param_list @@ -323,7 +324,7 @@ process_datatype(jdf_datatransfer_type_t *datatype, %type bodies %type VAR -%type EXTERN_DECL +%type EXTERN_DECL %type BODY_END %type ARROW %type PROPERTIES_ON @@ -372,23 +373,28 @@ jdf_file: prologue jdf epilogue current_jdf.epilogue = $3; } ; -prologue: EXTERN_DECL +prologue: EXTERN_DECL prologue { - $$ = new(jdf_external_entry_t); - $$->external_code = $1; - JDF_OBJECT_LINENO($$) = current_lineno; + jdf_external_entry_t *ent = new(jdf_external_entry_t); + ent->language = $1.language; + ent->external_code = $1.string; + ent->next = $2; + JDF_OBJECT_LINENO(ent) = current_lineno; + $$ = ent; } | { $$ = NULL; } ; -epilogue: EXTERN_DECL +epilogue: EXTERN_DECL epilogue { - $$ = new(jdf_external_entry_t); - $$->external_code = $1; - JDF_OBJECT_LINENO($$) = current_lineno; - } + jdf_external_entry_t *ent = new(jdf_external_entry_t); + ent->language = $1.language; + ent->external_code = $1.string; + ent->next = $2; + JDF_OBJECT_LINENO(ent) = current_lineno; + $$ = ent; } | { $$ = NULL; @@ -1325,7 +1331,7 @@ expr_simple: expr_simple EQUAL expr_simple { $$ = new(jdf_expr_t); $$->op = JDF_C_CODE; - $$->jdf_c_code.code = $1; + $$->jdf_c_code.code = $1.string; $$->jdf_type = PARSEC_RETURN_TYPE_INT32; $$->local_variables = current_locally_bound_variables; $$->scope = -1; diff --git a/parsec/mca/device/device.h b/parsec/mca/device/device.h index eeed22b20..730f82c56 100644 --- a/parsec/mca/device/device.h +++ b/parsec/mca/device/device.h @@ -66,6 +66,7 @@ typedef struct parsec_device_base_component_2_0_0 parsec_device_base_component_t #define PARSEC_DEV_INTEL_PHI ((uint8_t)(1 << 3)) #define PARSEC_DEV_OPENCL ((uint8_t)(1 << 4)) #define PARSEC_DEV_LEVEL_ZERO ((uint8_t)(1 << 5)) +#define PARSEC_DEV_DPCPP ((uint8_t)(1 << 5)) #define PARSEC_DEV_TEMPLATE ((uint8_t)(1 << 6)) #define PARSEC_DEV_ALL ((uint8_t) 0x1f) #define PARSEC_DEV_ANY_TYPE ((uint8_t) 0x1f) diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index d577636c9..31a53faac 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -72,9 +72,9 @@ parsec_gpu_check_space_needed(parsec_device_gpu_module_t *gpu_device, return space_needed; } -#if defined(PARSEC_PROF_TRACE) void parsec_gpu_init_profiling(void) { +#if defined(PARSEC_PROF_TRACE) if(parsec_gpu_profiling_initiated == 0) { parsec_profiling_add_dictionary_keyword("cuda", "fill:#66ff66", 0, NULL, @@ -100,8 +100,8 @@ void parsec_gpu_init_profiling(void) &parsec_gpu_use_memory_key_start, &parsec_gpu_use_memory_key_end); parsec_gpu_profiling_initiated = 1; } -} #endif +} void parsec_gpu_enable_debug(void) { diff --git a/parsec/mca/device/device_gpu.h b/parsec/mca/device/device_gpu.h index 10b333561..b664af9c8 100644 --- a/parsec/mca/device/device_gpu.h +++ b/parsec/mca/device/device_gpu.h @@ -227,8 +227,6 @@ extern int parsec_gpu_prefetch_key_start; extern int parsec_gpu_prefetch_key_end; extern int parsec_device_gpu_one_profiling_stream_per_gpu_stream; -void parsec_gpu_init_profiling(void); - typedef struct { uint64_t size; uint64_t data_key; @@ -237,6 +235,7 @@ typedef struct { #define PARSEC_DEVICE_GPU_MEMORY_PROF_INFO_CONVERTER "size{int64_t};data_key{uint64_t};dc_id{uint64_t}" #endif /* defined(PROFILING) */ +void parsec_gpu_init_profiling(void); void dump_exec_stream(parsec_gpu_exec_stream_t* exec_stream); void dump_GPU_state(parsec_device_gpu_module_t* gpu_device); diff --git a/parsec/mca/device/level_zero/ValidateModule.CMake b/parsec/mca/device/level_zero/ValidateModule.CMake index 88b5c21c0..1c2d08bd7 100644 --- a/parsec/mca/device/level_zero/ValidateModule.CMake +++ b/parsec/mca/device/level_zero/ValidateModule.CMake @@ -1,15 +1,37 @@ # For now assume that the upper level did the CUDA search and that all # necessary dependencies on CUDA have been correctly setup -if( PARSEC_HAVE_LEVEL_ZERO ) +if( PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP ) SET(MCA_${COMPONENT}_${MODULE} ON) FILE(GLOB MCA_${COMPONENT}_${MODULE}_SOURCES ${MCA_BASE_DIR}/${COMPONENT}/${MODULE}/[^\\.]*.c) + ADD_CUSTOM_COMMAND(OUTPUT ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o + MAIN_DEPENDENCY ${PROJECT_SOURCE_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero + COMMAND ${DPCPP_EXECUTABLE} + -I${LEVEL_ZERO_INCLUDE_DIR} + -I${SYCL_INCLUDE_DIR} + -I${PROJECT_SOURCE_DIR} + -I${PROJECT_SOURCE_DIR}/parsec/include + -I${PROJECT_BINARY_DIR}/parsec/include + -I${PROJET_BINARY_DIR} + -Wall -fpic -g -c ${PROJECT_SOURCE_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp) + SET_SOURCE_FILES_PROPERTIES( + ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o + PROPERTIES + EXTERNAL_OBJECT true + GENERATED true) + add_library(parsec-dpcpp ${PROJECT_BINARY_DIR}/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.o) + target_link_libraries(parsec PRIVATE parsec-dpcpp ${SYCL_LIBRARY}) + install(TARGETS parsec-dpcpp + EXPORT parsec-targets + DESTINATION ${PARSEC_INSTALL_LIBDIR}) SET(MCA_${COMPONENT}_${MODULE}_CONSTRUCTOR "${COMPONENT}_${MODULE}_static_component") install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero.h ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero_internal.h + ${CMAKE_CURRENT_SOURCE_DIR}/mca/device/level_zero/device_level_zero_dpcpp.h DESTINATION include/parsec/mca/device/level_zero ) -else (PARSEC_HAVE_LEVEL_ZERO) - MESSAGE(STATUS "Module ${MODULE} not selectable: does not have LEVEL_ZERO") +else (PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP) + MESSAGE(STATUS "Module ${MODULE} not selectable: does not have LEVEL_ZERO (${PARSEC_HAVE_LEVEL_ZERO}) or DPCPP (${PARSEC_HAVE_DPCPP})") SET(MCA_${COMPONENT}_${MODULE} OFF) -endif(PARSEC_HAVE_LEVEL_ZERO) +endif(PARSEC_HAVE_LEVEL_ZERO AND PARSEC_HAVE_DPCPP) diff --git a/parsec/mca/device/level_zero/device_level_zero.h b/parsec/mca/device/level_zero/device_level_zero.h index 2937b46fb..19c13715b 100644 --- a/parsec/mca/device/level_zero/device_level_zero.h +++ b/parsec/mca/device/level_zero/device_level_zero.h @@ -7,17 +7,12 @@ #ifndef PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED #define PARSEC_DEVICE_LEVEL_ZERO_H_HAS_BEEN_INCLUDED -#include "parsec/parsec_internal.h" -#include "parsec/class/parsec_object.h" -#include "parsec/mca/device/device.h" #if defined(PARSEC_HAVE_LEVEL_ZERO) -#include "parsec/class/list_item.h" -#include "parsec/class/list.h" -#include "parsec/class/fifo.h" +#include "parsec/mca/device/device.h" #include "parsec/mca/device/device_gpu.h" -#include +#include BEGIN_C_DECLS @@ -59,18 +54,63 @@ struct parsec_level_zero_exec_stream_s { ze_event_handle_t *events; ze_event_pool_handle_t ze_event_pool; ze_command_list_handle_t level_zero_cl; + ze_command_queue_handle_t level_zero_cq; + void *dpcpp_obj; }; + +/** +* Progress +*/ +/** + * This version is based on 4 streams: one for transfers from the memory to + * the GPU, 2 for kernel executions and one for transfers from the GPU into + * the main memory. The synchronization on each stream is based on GPU events, + * such an event indicate that a specific epoch of the lifetime of a task has + * been completed. Each type of stream (in, exec and out) has a pending FIFO, + * where tasks ready to jump to the respective step are waiting. + */ +parsec_hook_return_t +parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, + parsec_gpu_task_t *gpu_task, + int which_gpu ); + /**************************************************** ** GPU-DATA Specific Starts Here ** ****************************************************/ -PARSEC_DECLSPEC extern int parsec_level_zero_output_stream; - /** * Overload the default data_copy_t with a GPU specialized type */ typedef parsec_data_copy_t parsec_gpu_data_copy_t; +/* Default stage_in function to transfer data to the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); + +/* Default stage_out function to transfer data from the GPU device. + * Transfer transfer the contiguous bytes from + * task->data[i].data_in to task->data[i].data_out. + * + * @param[in] task parsec_task_t containing task->data[i].data_in, task->data[i].data_out. + * @param[in] flow_mask indicating task flows for which to transfer. + * @param[in] gpu_stream parsec_gpu_exec_stream_t used for the transfer. + * + */ +int +parsec_default_level_zero_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream); + END_C_DECLS #define PARSEC_LEVEL_ZERO_CHECK_ERROR(STR, ERROR, CODE) \ diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c index 989ed8214..bac2c57ee 100644 --- a/parsec/mca/device/level_zero/device_level_zero_component.c +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -23,7 +23,7 @@ #include "parsec/utils/zone_malloc.h" #include "parsec/class/fifo.h" -#include +#include PARSEC_OBJ_CLASS_INSTANCE(parsec_device_level_zero_module_t, parsec_device_module_t, NULL, NULL); @@ -33,6 +33,7 @@ static int device_level_zero_component_query(mca_base_module_2_0_0_t **module, i static int device_level_zero_component_register(void); int use_level_zero_index, use_level_zero; +int parsec_cuda_sort_pending = 0, parsec_cuda_max_streams = PARSEC_GPU_MAX_STREAMS; int level_zero_mask, level_zero_nvlink_mask; int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; @@ -187,7 +188,7 @@ static int device_level_zero_component_register(void) false, false, -1, &level_zero_memory_number_of_blocks); (void)parsec_mca_param_reg_int_name("device_level_zero", "max_number_of_ejected_data", "Sets up the maximum number of blocks that can be ejected from GPU memory", - false, false, MAX_PARAM_COUNT, &parsec_GPU_d2h_max_flows); + false, false, MAX_PARAM_COUNT, &parsec_gpu_d2h_max_flows); (void)parsec_mca_param_reg_int_name("device_level_zero", "sort_pending_tasks", "Boolean to let the GPU engine sort the first pending tasks stored in the list", false, false, 0, &parsec_LEVEL_ZERO_sort_pending_list); @@ -196,7 +197,12 @@ static int device_level_zero_component_register(void) "Boolean to separate the profiling of each level_zero stream into a single profiling stream", false, false, 0, &parsec_device_gpu_one_profiling_stream_per_gpu_stream); #endif - + (void)parsec_mca_param_reg_int_name("device_level_zero", "max_streams", + "Maximum number of Streams to use for the GPU engine; 2 streams are used for communication between host and device, so the minimum is 3", + false, false, PARSEC_GPU_MAX_STREAMS, &parsec_level_zero_max_streams); + (void)parsec_mca_param_reg_int_name("device_level_zero", "sort_pending_tasks", + "Boolean to let the GPU engine sort the first pending tasks stored in the list", + false, false, 0, &parsec_level_zero_sort_pending); /* If LEVEL_ZERO was not requested avoid initializing the devices */ return (0 == use_level_zero ? MCA_ERROR : MCA_SUCCESS); } diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp.h b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h new file mode 100644 index 000000000..993cd0bfe --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h @@ -0,0 +1,27 @@ +#ifndef PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H +#define PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H + +#if defined(c_plusplus) || defined(__cplusplus) +#include "sycl/backend/level_zero.hpp" + +typedef struct { + sycl::platform platform; + sycl::device device; + sycl::context context; + sycl::queue queue; +} parsec_dpcpp_object_t; + +extern "C" { +#endif + +void * parsec_dpcpp_queue_create(ze_driver_handle_t ze_driver, + ze_device_handle_t ze_device, + ze_context_handle_t ze_context, + ze_command_queue_handle_t ze_queue); +int parsec_dpcpp_queue_destroy(void *_dpcpp_obj); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif //PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp b/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp new file mode 100644 index 000000000..0868e9911 --- /dev/null +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp @@ -0,0 +1,28 @@ +#include "level_zero/ze_api.h" +#include "device_level_zero_dpcpp.h" + +void * parsec_dpcpp_queue_create(ze_driver_handle_t ze_driver, + ze_device_handle_t ze_device, + ze_context_handle_t ze_context, + ze_command_queue_handle_t ze_queue) +{ + parsec_dpcpp_object_t *dpcpp_obj; + std::vectordevices; + + dpcpp_obj = new parsec_dpcpp_object_t; + dpcpp_obj->platform = sycl::level_zero::make(ze_driver); + dpcpp_obj->device = sycl::level_zero::make(dpcpp_obj->platform, ze_device); + devices.push_back(dpcpp_obj->device); + dpcpp_obj->context = sycl::level_zero::make(devices, ze_context); + dpcpp_obj->queue = sycl::level_zero::make(dpcpp_obj->context, ze_queue); + + return static_cast(dpcpp_obj); +} + +int parsec_dpcpp_queue_destroy(void *_dpcpp_obj) +{ + parsec_dpcpp_object_t *dpcpp_obj = reinterpret_cast(_dpcpp_obj); + delete dpcpp_obj; + + return 0; +} diff --git a/parsec/mca/device/level_zero/device_level_zero_internal.h b/parsec/mca/device/level_zero/device_level_zero_internal.h index c9c8725f9..e6ff2cf16 100644 --- a/parsec/mca/device/level_zero/device_level_zero_internal.h +++ b/parsec/mca/device/level_zero/device_level_zero_internal.h @@ -46,6 +46,7 @@ extern int parsec_device_level_zero_one_profiling_stream_per_level_zero_stream; /* From MCA parameters */ extern int use_level_zero_index, use_level_zero; extern int level_zero_mask, level_zero_verbosity; +extern int parsec_level_zero_sort_pending, parsec_level_zero_max_streams; extern int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; extern char* level_zero_lib_path; extern int32_t parsec_LEVEL_ZERO_d2h_max_flows; diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c index bbb0c3771..68547513c 100644 --- a/parsec/mca/device/level_zero/device_level_zero_module.c +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 The University of Tennessee and The University + * Copyright (c) 2021-2022 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. */ @@ -23,8 +23,9 @@ #include "parsec/utils/argv.h" #include "parsec/utils/zone_malloc.h" #include "parsec/class/fifo.h" +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" -#include +#include static int parsec_level_zero_data_advise(parsec_device_module_t *dev, parsec_data_t *data, int advice); static int @@ -199,7 +200,7 @@ int parsec_level_zero_module_init( int dev_id, ze_driver_handle_t ze_driver, ze_ gpu_device->super.name = ""; gpu_device->data_avail_epoch = 0; - gpu_device->max_exec_streams = PARSEC_MAX_STREAMS; + gpu_device->max_exec_streams = parsec_level_zero_max_streams; gpu_device->exec_stream = (parsec_gpu_exec_stream_t**)malloc(gpu_device->max_exec_streams * sizeof(parsec_gpu_exec_stream_t*)); gpu_device->exec_stream[0] = @@ -254,15 +255,35 @@ int parsec_level_zero_module_init( int dev_id, ze_driver_handle_t ze_driver, ze_ ZE_COMMAND_QUEUE_MODE_DEFAULT, ZE_COMMAND_QUEUE_PRIORITY_NORMAL }; - if( j < 2 ) + if( j < 2 ) { commandQueueDesc.ordinal = copyQueueGroupOrdinal; - else + ze_rc = zeCommandListCreateImmediate(level_zero_device->ze_context, level_zero_device->ze_device, + &commandQueueDesc, + &level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreateImmediate ", ze_rc, + {goto release_device;} ); + } else { commandQueueDesc.ordinal = computeQueueGroupOrdinal; - ze_rc = zeCommandListCreateImmediate(level_zero_device->ze_context, level_zero_device->ze_device, - &commandQueueDesc, - &level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreateImmediate ", ze_rc, - {goto release_device;} ); + ze_rc = zeCommandQueueCreate(level_zero_device->ze_context, level_zero_device->ze_device, + &commandQueueDesc, &level_zero_stream->level_zero_cq); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, + {goto release_device;} ); + ze_command_list_desc_t commandListDesc = { + ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, + NULL, + computeQueueGroupOrdinal, + 0 // flags + }; + ze_rc = zeCommandListCreate(level_zero_device->ze_context, level_zero_device->ze_device, + &commandListDesc, &level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, + {goto release_device;} ); + level_zero_stream->dpcpp_obj = parsec_dpcpp_queue_create(level_zero_device->ze_driver, + level_zero_device->ze_device, + level_zero_device->ze_context, + level_zero_stream->level_zero_cq); + assert(NULL != level_zero_stream->dpcpp_obj); + } exec_stream->workspace = NULL; PARSEC_OBJ_CONSTRUCT(&exec_stream->infos, parsec_info_object_array_t); parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream); @@ -338,9 +359,9 @@ int parsec_level_zero_module_init( int dev_id, ze_driver_handle_t ze_driver, ze_ device->type = PARSEC_DEV_LEVEL_ZERO; device->executed_tasks = 0; - device->transferred_data_in = 0; - device->d2d_transfer = 0; - device->transferred_data_out = 0; + device->data_in_array_size = 0; // We'll let the modules_attach allocate the array of the right size for us + device->data_in_from_device = NULL; + device->data_out_to_host = 0; device->required_data_in = 0; device->required_data_out = 0; @@ -475,7 +496,17 @@ parsec_level_zero_module_fini(parsec_device_module_t* device) free(exec_stream->tasks); exec_stream->tasks = NULL; free(exec_stream->fifo_pending); exec_stream->fifo_pending = NULL; /* Release the stream */ - zeCommandListDestroy( level_zero_stream->level_zero_cl ); + if(k < 2) { + ze_result_t ze_rc; + ze_rc = zeCommandListDestroy(level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListDestroy ", ze_rc, + {continue;} ); + } else { + int rc; + rc = parsec_dpcpp_queue_destroy(level_zero_stream->dpcpp_obj); + assert(PARSEC_SUCCESS == rc); + level_zero_stream->dpcpp_obj = NULL; + } free(exec_stream->name); /* Release Info object array */ @@ -1100,9 +1131,9 @@ parsec_gpu_data_reserve_device_space( parsec_device_level_zero_module_t* level_z * */ int -parsec_level_zero_stage_in(parsec_gpu_task_t *gtask, - uint32_t flow_mask, - parsec_gpu_exec_stream_t *gpu_stream) +parsec_default_level_zero_stage_in(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) { ze_result_t ret; parsec_data_copy_t * copy_in; @@ -1139,9 +1170,9 @@ parsec_level_zero_stage_in(parsec_gpu_task_t *gtask, * */ int -parsec_level_zero_stage_out(parsec_gpu_task_t *gtask, - uint32_t flow_mask, - parsec_gpu_exec_stream_t *gpu_stream) +parsec_default_level_zero_stage_out(parsec_gpu_task_t *gtask, + uint32_t flow_mask, + parsec_gpu_exec_stream_t *gpu_stream) { ze_result_t ret; parsec_data_copy_t * copy_in; @@ -1394,12 +1425,9 @@ parsec_gpu_data_stage_in( parsec_device_level_zero_module_t* level_zero_device, assert(0); return -1; } - - if( in_elem_dev->super.super.type != PARSEC_DEV_LEVEL_ZERO ) - gpu_device->super.transferred_data_in += nb_elts; - else - gpu_device->super.d2d_transfer += nb_elts; - if( GPU_TASK_TYPE_KERNEL == gpu_task->task_type ) + assert(in_elem_dev->super.super.device_index < gpu_device->super.data_in_array_size); + gpu_device->super.data_in_from_device[in_elem_dev->super.super.device_index] += nb_elts; + if( PARSEC_GPU_TASK_TYPE_KERNEL == gpu_task->task_type ) gpu_device->super.nb_data_faults += nb_elts; /* update the data version in GPU immediately, and mark the data under transfer */ @@ -1460,7 +1488,6 @@ static parsec_task_class_t parsec_level_zero_data_prefetch_tc = { .initial_data = NULL, .final_data = NULL, .data_affinity = NULL, - .key_generator = NULL, .key_functions = NULL, .make_key = NULL, .get_datatype = NULL, @@ -1558,8 +1585,8 @@ parsec_level_zero_data_advise(parsec_device_module_t *dev, parsec_data_t *data, gpu_task->ec->task_class = &parsec_level_zero_data_prefetch_tc; gpu_task->flow[0] = &parsec_level_zero_data_prefetch_flow; gpu_task->flow_nb_elts[0] = data->device_copies[ data->owner_device ]->original->nb_elts; - gpu_task->stage_in = parsec_level_zero_stage_in; - gpu_task->stage_out = parsec_level_zero_stage_out; + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; PARSEC_DEBUG_VERBOSE(20, parsec_debug_output, "Retain data copy %p [ref_count %d] at %s:%d", data->device_copies[ data->owner_device ], data->device_copies[ data->owner_device ]->super.super.obj_reference_count, @@ -1652,8 +1679,8 @@ parsec_gpu_send_transfercomplete_cmd_to_device(parsec_data_copy_t *copy, gpu_task->ec->task_class = &parsec_level_zero_d2d_complete_tc; gpu_task->flow[0] = &parsec_level_zero_d2d_complete_flow; gpu_task->flow_nb_elts[0] = copy->original->nb_elts; - gpu_task->stage_in = parsec_level_zero_stage_in; - gpu_task->stage_out = parsec_level_zero_stage_out; + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; gpu_task->ec->data[0].data_in = copy; /* We need to set not-null in data_in, so that the fake flow is * not ignored when poping the data from the fake task */ gpu_task->ec->data[0].data_out = copy; /* We "free" data[i].data_out if its readers reaches 0 */ @@ -1677,7 +1704,6 @@ parsec_gpu_callback_complete_push(parsec_device_gpu_module_t *gpu_device, parsec_gpu_task_t **gpu_task, parsec_gpu_exec_stream_t *gpu_stream) { - parsec_device_level_zero_module_t* level_zero_device = (parsec_device_level_zero_module_t*)gpu_device; (void)gpu_stream; parsec_gpu_task_t *gtask = *gpu_task; @@ -1869,11 +1895,11 @@ parsec_gpu_callback_complete_push(parsec_device_gpu_module_t *gpu_device, static inline int progress_stream( parsec_device_gpu_module_t* gpu_device, parsec_gpu_exec_stream_t* stream, - advance_task_function_t upstream_progress_fct, + parsec_advance_task_function_t upstream_progress_fct, parsec_gpu_task_t* task, parsec_gpu_task_t** out_task ) { - advance_task_function_t progress_fct; + parsec_advance_task_function_t progress_fct; int saved_rc = 0, rc; #if defined(PARSEC_DEBUG_NOISIER) char task_str[MAX_TASK_STRLEN]; @@ -1983,6 +2009,11 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, */ rc = zeCommandListAppendSignalEvent( level_zero_stream->level_zero_cl, level_zero_stream->events[stream->start] ); assert(ZE_RESULT_SUCCESS == rc); + if( stream->start >= 2 ) { + ze_result_t ze_rc = zeCommandListClose(level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, + { } ); + } stream->tasks[stream->start] = task; stream->start = (stream->start + 1) % stream->max_events; PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, @@ -2246,7 +2277,7 @@ parsec_gpu_kernel_pop( parsec_device_gpu_module_t *gpu_device, parsec_atomic_unlock(&original->lock); goto release_and_return_error; } - gpu_device->super.transferred_data_out += nb_elts; /* TODO: not hardcoded, use datatype size */ + gpu_device->super.data_out_to_host += nb_elts; /* TODO: not hardcoded, use datatype size */ how_many++; } else { assert( 0 == gpu_copy->readers ); @@ -2526,7 +2557,7 @@ parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, progress_task = NULL; } /* If we can extract data go for it, otherwise try to drain the pending tasks */ - gpu_task = parsec_gpu_create_W2R_task(gpu_device, es); + gpu_task = parsec_gpu_create_w2r_task(gpu_device, es); if( NULL != gpu_task ) goto get_data_out_of_device; } @@ -2593,7 +2624,7 @@ parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, fetch_task_from_shared_queue: assert( NULL == gpu_task ); - if (1 == parsec_LEVEL_ZERO_sort_pending_list && out_task_submit == NULL && out_task_pop == NULL) { + if (1 == parsec_level_zero_sort_pending && out_task_submit == NULL && out_task_pop == NULL) { parsec_gpu_sort_pending_list(gpu_device); } gpu_task = (parsec_gpu_task_t*)parsec_fifo_try_pop( &(gpu_device->pending) ); @@ -2623,7 +2654,7 @@ parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, /* Everything went fine so far, the result is correct and back in the main memory */ PARSEC_LIST_ITEM_SINGLETON(gpu_task); if (gpu_task->task_type == GPU_TASK_TYPE_D2HTRANSFER) { - parsec_gpu_W2R_task_fini(gpu_device, gpu_task, es); + parsec_gpu_w2r_task_fini(gpu_device, gpu_task, es); gpu_task = progress_task; goto fetch_task_from_shared_queue; } diff --git a/tests/dsl/ptg/cuda/nvlink.jdf b/tests/dsl/ptg/cuda/nvlink.jdf index 2402b5cfe..70397a934 100644 --- a/tests/dsl/ptg/cuda/nvlink.jdf +++ b/tests/dsl/ptg/cuda/nvlink.jdf @@ -38,6 +38,13 @@ typedef cublasStatus_t (*cublas_dgemm_v2_t) ( cublasHandle_t handle, %} +extern "DPCPP" %{ + +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include + +%} + %option no_taskpool_instance = true /* can be anything */ /* @@ -150,6 +157,19 @@ BODY [type=CUDA } END +BODY [type=DPCPP + weight=(1)] +{ + double alpha=0.0; + double beta=1.0; + oneapi::mkl::blas::gemm(parsec_dpcpp->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + descA->super.mb, descA->super.nb, descA->super.mb, + alpha, static_cast(A), descA->super.mb, + static_cast(A), descA->super.mb, + beta, static_cast(C), descA->super.mb); +} +END + BODY { fprintf(stderr, "Kernel GEMM(%d, %d, %d) in nvlink test is running on a CPU, which is not the intended behavior\n", From ce47168443db96b1b27e77645966350a50122c10 Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Wed, 12 Oct 2022 22:31:13 +0000 Subject: [PATCH 03/15] Re-enable Level Zero test; update to latest level zero / oneAPI API --- parsec/interfaces/ptg/ptg-compiler/jdf2c.c | 6 +++--- .../level_zero/device_level_zero_dpcpp.h | 2 +- parsec/parsec_internal.h | 4 ++++ tests/dsl/ptg/cuda/CMakeLists.txt | 20 +++++++++++++++++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index 72342d266..37b59f268 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -7160,13 +7160,13 @@ static void jdf_generate_code_hook_dpcpp(const jdf_t *jdf, " if (dev_index < -1) {\n" " return PARSEC_HOOK_RETURN_NEXT;\n" " } else if (dev_index == -1) {\n" - " dev_index = parsec_gpu_get_best_device((parsec_task_t*)this_task, ratio);\n" + " dev_index = parsec_get_best_device((parsec_task_t*)this_task, ratio);\n" " } else {\n" " dev_index = (dev_index %% (parsec_mca_device_enabled()-2)) + 2;\n" " }\n", device); } else { - coutput(" dev_index = parsec_gpu_get_best_device((parsec_task_t*)this_task, ratio);\n"); + coutput(" dev_index = parsec_get_best_device((parsec_task_t*)this_task, ratio);\n"); } coutput(" assert(dev_index >= 0);\n" " if( dev_index < 2 ) {\n" @@ -8805,7 +8805,7 @@ int jdf2c(const char *output_c, const char *output_h, dpcpp_output("#include \"parsec.h\"\n" "#include \"level_zero/ze_api.h\"\n" - "#include \"sycl/backend/level_zero.hpp\"\n" + "#include \"sycl/ext/oneapi/backend/level_zero.hpp\"\n" "#include \"parsec/parsec_internal.h\"\n" "#include \"parsec/execution_stream.h\"\n" "#include \"parsec/mca/device/device.h\"\n" diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp.h b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h index 993cd0bfe..00c00cc63 100644 --- a/parsec/mca/device/level_zero/device_level_zero_dpcpp.h +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h @@ -2,7 +2,7 @@ #define PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H #if defined(c_plusplus) || defined(__cplusplus) -#include "sycl/backend/level_zero.hpp" +#include "sycl/ext/oneapi/backend/level_zero.hpp" typedef struct { sycl::platform platform; diff --git a/parsec/parsec_internal.h b/parsec/parsec_internal.h index 3beb1310f..c4815b4bb 100644 --- a/parsec/parsec_internal.h +++ b/parsec/parsec_internal.h @@ -334,6 +334,10 @@ typedef int (parsec_data_ref_fn_t)(parsec_task_t *task, #define PARSEC_TASK_CLASS_TYPE_PTG 0x01 #define PARSEC_TASK_CLASS_TYPE_DTD 0x02 +#if defined(__cplusplus) +#define restrict +#endif + /** * Find the dependency corresponding to a given execution context. */ diff --git a/tests/dsl/ptg/cuda/CMakeLists.txt b/tests/dsl/ptg/cuda/CMakeLists.txt index d4d73b014..6ce518e9f 100644 --- a/tests/dsl/ptg/cuda/CMakeLists.txt +++ b/tests/dsl/ptg/cuda/CMakeLists.txt @@ -28,3 +28,23 @@ if(PARSEC_HAVE_CUDA) parsec_addtest_cmd(get_best_device_shm ${SHM_TEST_CMD_LIST} ./testing_get_best_device -N 400 -t 20 -g 1) endif(PARSEC_HAVE_CUDA) + +if(PARSEC_HAVE_LEVEL_ZERO) + if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) + endif(NOT TARGET MKL::MKL_DPCPP) + if(TARGET MKL::MKL_DPCPP) + message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + include(ParsecCompilePTG) + + parsec_addtest_executable(C nvlink SOURCES nvlink_main.c nvlink_wrapper.c) + target_include_directories(nvlink PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) + target_ptg_sources(nvlink PRIVATE "nvlink.jdf") + target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP) + else(TARGET MKL::MKL_DPCPP) + message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink test with DPCPP is disabled") + endif(TARGET MKL::MKL_DPCPP) +endif(PARSEC_HAVE_LEVEL_ZERO) From 644caed0b743700240a582a4d6d9bb2c690cfa48 Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Thu, 13 Oct 2022 17:57:43 +0000 Subject: [PATCH 04/15] Update wrapper to allow testing both CUDA and Level Zero with new Level Zero update use_cuda / use_cuda_index have been renamed to follow proper naming scheme; do the same for level_zero --- .../level_zero/device_level_zero_component.c | 46 +++++----- tests/dsl/ptg/cuda/nvlink_wrapper.c | 85 +++++++++++++++---- 2 files changed, 91 insertions(+), 40 deletions(-) diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c index bac2c57ee..db78be3d6 100644 --- a/parsec/mca/device/level_zero/device_level_zero_component.c +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -91,12 +91,12 @@ static int device_level_zero_component_query(mca_base_module_t **module, int *pr *module = NULL; *priority = 0; - if( 0 == use_level_zero ) { + if( 0 == parsec_device_level_zero_enabled ) { return MCA_SUCCESS; } parsec_gpu_init_profiling(); - if( use_level_zero >= 1) { + if( parsec_device_level_zero_enabled >= 1) { uint32_t driverCount = 0; uint32_t totalDeviceCount = 0, maxDeviceCount = 0; @@ -114,19 +114,19 @@ static int device_level_zero_component_query(mca_base_module_t **module, int *pr maxDeviceCount = deviceCount; } - use_level_zero = totalDeviceCount < (uint32_t)use_level_zero ? (int)totalDeviceCount : use_level_zero; + parsec_device_level_zero_enabled = totalDeviceCount < (uint32_t)parsec_device_level_zero_enabled ? (int)totalDeviceCount : parsec_device_level_zero_enabled; - if(use_level_zero > 0) { + if(parsec_device_level_zero_enabled > 0) { parsec_device_level_zero_component.modules = - (parsec_device_module_t **)calloc(use_level_zero + 1, + (parsec_device_module_t **)calloc(parsec_device_level_zero_enabled + 1, sizeof(parsec_device_module_t *)); devices = (ze_device_handle_t *)malloc(maxDeviceCount * sizeof(ze_device_handle_t)); i = j = 0; - for(uint32_t did = 0; i < use_level_zero && did < driverCount; ++did ) { + for(uint32_t did = 0; i < parsec_device_level_zero_enabled && did < driverCount; ++did ) { uint32_t deviceCount = maxDeviceCount; zeDeviceGet(allDrivers[did], &deviceCount, devices); - for(uint32_t devid = 0; i < use_level_zero && devid < deviceCount; devid++) { + for(uint32_t devid = 0; i < parsec_device_level_zero_enabled && devid < deviceCount; devid++) { ze_device_properties_t device_properties; zeDeviceGetProperties(devices[devid], &device_properties); if( ZE_DEVICE_TYPE_GPU != device_properties.type) { continue; } @@ -165,9 +165,9 @@ static int device_level_zero_component_query(mca_base_module_t **module, int *pr static int device_level_zero_component_register(void) { - use_level_zero_index = parsec_mca_param_reg_int_name("device_level_zero", "enabled", + parsec_device_level_zero_index = parsec_mca_param_reg_int_name("device_level_zero", "enabled", "The number of LEVEL_ZERO device to enable for the next PaRSEC context (-1 for all available)", - false, false, -1, &use_level_zero); + false, false, -1, &parsec_device_level_zero_enabled); (void)parsec_mca_param_reg_int_name("device_level_zero", "mask", "The bitwise mask of LEVEL_ZERO devices to be enabled (default all)", false, false, 0xffffffff, &level_zero_mask); @@ -204,7 +204,7 @@ static int device_level_zero_component_register(void) "Boolean to let the GPU engine sort the first pending tasks stored in the list", false, false, 0, &parsec_level_zero_sort_pending); /* If LEVEL_ZERO was not requested avoid initializing the devices */ - return (0 == use_level_zero ? MCA_ERROR : MCA_SUCCESS); + return (0 == parsec_device_level_zero_enabled ? MCA_ERROR : MCA_SUCCESS); } /** @@ -219,14 +219,14 @@ static int device_level_zero_component_open(void) int ndevices = 0; uint32_t driverCount = 0; - if( 0 <= use_level_zero ) { + if( 0 == parsec_device_level_zero_enabled ) { return MCA_ERROR; /* Nothing to do around here */ } ze_rc = zeInit( 0 ); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeInit ", ze_rc, { - parsec_mca_param_set_int(use_level_zero_index, 0); + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); return MCA_ERROR; } ); @@ -234,7 +234,7 @@ static int device_level_zero_component_open(void) ze_rc = zeDriverGet(&driverCount, NULL); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, { - parsec_mca_param_set_int(use_level_zero_index, 0); + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); return MCA_ERROR; } ); if(driverCount > 0) { @@ -243,7 +243,7 @@ static int device_level_zero_component_open(void) PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, { free(allDrivers); - parsec_mca_param_set_int(use_level_zero_index, 0); + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); return MCA_ERROR; } ); @@ -253,7 +253,7 @@ static int device_level_zero_component_open(void) PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet ", ze_rc, { free(allDrivers); - parsec_mca_param_set_int(use_level_zero_index, 0); + parsec_mca_param_set_int(parsec_device_level_zero_index, 0); return MCA_ERROR; } ); ndevices += (int)deviceCount; @@ -263,21 +263,21 @@ static int device_level_zero_component_open(void) } - if( ndevices > use_level_zero ) { - if( 0 < use_level_zero_index ) { - ndevices = use_level_zero; + if( ndevices > parsec_device_level_zero_enabled ) { + if( 0 < parsec_device_level_zero_index ) { + ndevices = parsec_device_level_zero_enabled; } - } else if (ndevices < use_level_zero ) { - if( 0 < use_level_zero_index ) { + } else if (ndevices < parsec_device_level_zero_enabled ) { + if( 0 < parsec_device_level_zero_index ) { parsec_warning("User requested %d LEVEL_ZERO devices, but only %d are available on %s\n." " PaRSEC will enable all %d of them.", - use_level_zero, ndevices, parsec_hostname, ndevices); - parsec_mca_param_set_int(use_level_zero_index, ndevices); + parsec_device_level_zero_enabled, ndevices, parsec_hostname, ndevices); + parsec_mca_param_set_int(parsec_device_level_zero_index, ndevices); } } /* Update the number of GPU for the upper layer */ - use_level_zero = ndevices; + parsec_device_level_zero_enabled = ndevices; if( 0 == ndevices ) { return -1; } diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index 06a34079a..a5ae5324d 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -6,7 +6,6 @@ */ #include "parsec.h" -#include "parsec/mca/device/cuda/device_cuda_internal.h" #include "parsec/data_distribution.h" #include "parsec/data_dist/matrix/matrix.h" #include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" @@ -14,12 +13,20 @@ #include "parsec/class/info.h" #if defined(PARSEC_HAVE_CUDA) +#include "parsec/mca/device/cuda/device_cuda_internal.h" #include +#define TARGET_DEVICE_TYPE PARSEC_DEV_CUDA +#elif defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" +#define TARGET_DEVICE_TYPE PARSEC_DEV_LEVEL_ZERO +#else +#define TARGET_DEVICE_TYPE PARSEC_DEV_CPU #endif #include "nvlink.h" #if defined(PARSEC_HAVE_CUDA) +/* Only CUDA/CUBLAS requires to define handles on each GPU device */ static void destruct_cublas_handle(void *p) { cublasHandle_t handle = (cublasHandle_t)p; @@ -45,17 +52,15 @@ static void *create_cublas_handle(void *obj, void *p) (void)status; return (void*)handle; } -#endif static void destroy_cublas_handle(void *_h, void *_n) { -#if defined(PARSEC_HAVE_CUDA) cublasHandle_t cublas_handle = (cublasHandle_t)_h; cublasDestroy_v2(cublas_handle); -#endif (void)_n; (void)_h; } +#endif static void __parsec_nvlink_destructor( parsec_nvlink_taskpool_t* nvlink_taskpool) @@ -65,22 +70,33 @@ __parsec_nvlink_destructor( parsec_nvlink_taskpool_t* nvlink_taskpool) parsec_matrix_block_cyclic_t *dcA; parsec_del2arena( & nvlink_taskpool->arenas_datatypes[PARSEC_nvlink_DEFAULT_ADT_IDX] ); parsec_data_free(nvlink_taskpool->_g_descA->mat); +#if defined(PARSEC_HAVE_CUDA) parsec_info_unregister(&parsec_per_stream_infos, nvlink_taskpool->_g_CuHI, NULL); +#endif dcA = nvlink_taskpool->_g_descA; parsec_tiled_matrix_destroy( (parsec_tiled_matrix_t*)nvlink_taskpool->_g_descA ); userM = nvlink_taskpool->_g_userM; for(g = 0, dev = 0; dev < (int)parsec_nb_devices; dev++) { - parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == cuda_device->super.super.type ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( TARGET_DEVICE_TYPE == device->type ) { parsec_data_t *dta = ((parsec_dc_t*)userM)->data_of((parsec_dc_t*)userM, g, userM->super.super.myrank); - parsec_data_copy_t *gpu_copy = parsec_data_get_copy(dta, cuda_device->super.super.device_index); + parsec_data_copy_t *gpu_copy = parsec_data_get_copy(dta, device->device_index); +#if defined(PARSEC_HAVE_CUDA) + parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)device; cudaError_t status = cudaSetDevice( cuda_device->cuda_index ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaSetDevice ", status, {} ); status = (cudaError_t)cudaFree( gpu_copy->device_private ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaFree ", status, {} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; + ze_result_t status = zeMemFree(level_zero_device->ze_context, gpu_copy->device_private); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, {} ); +#else + free(gpu_copy->device_private); +#endif gpu_copy->device_private = NULL; - parsec_data_copy_detach(dta, gpu_copy, cuda_device->super.super.device_index); + parsec_data_copy_detach(dta, gpu_copy, device->device_index); PARSEC_OBJ_RELEASE(gpu_copy); g++; } @@ -101,18 +117,18 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb parsec_matrix_block_cyclic_t *dcA; parsec_matrix_block_cyclic_t *userM; - /** Find all CUDA devices */ + /** Find all GPU devices */ nb = 0; for(dev = 0; dev < (int)parsec_nb_devices; dev++) { parsec_device_module_t *device = parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == device->type ) { + if( TARGET_DEVICE_TYPE == device->type ) { nb++; } } if(nb == 0) { char hostname[256]; gethostname(hostname, 256); - fprintf(stderr, "This test requires at least one CUDA device per node -- no CUDA device found on rank %d on %s\n", + fprintf(stderr, "This test requires at least one GPU device per node -- no GPU device found on rank %d on %s\n", ctx->my_rank, hostname); return NULL; } @@ -120,7 +136,7 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb nb = 0; for(dev = 0; dev < (int)parsec_nb_devices; dev++) { parsec_device_module_t *device = parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == device->type ) { + if( TARGET_DEVICE_TYPE == device->type ) { dev_index[nb++] = device->device_index; } } @@ -179,31 +195,66 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb /* Now, we create a GPU version of each tile. As these tiles will be accessed RW * in the JDF, this also pins the task on the GPU that we chose to host the tile */ for(int g = 0, dev = 0; dev < (int)parsec_nb_devices; dev++) { - parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t*)parsec_mca_device_get(dev); - if( PARSEC_DEV_CUDA == cuda_device->super.super.type ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( TARGET_DEVICE_TYPE == device->type ) { /* We get the data from the data collection */ parsec_data_t *dta = ((parsec_dc_t*)userM)->data_of((parsec_dc_t*)userM, g, ctx->my_rank); /* The corresponding data copy on CPU RAM */ parsec_data_copy_t *cpu_copy = parsec_data_get_copy(dta, 0); /* And we create a new data copy on GPU */ parsec_data_copy_t *gpu_copy = PARSEC_OBJ_NEW(parsec_data_copy_t); - /* We chose the GPU */ +#if defined(PARSEC_HAVE_CUDA) + /* We chose the GPU */ + parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t *)device; cudaError_t status = cudaSetDevice( cuda_device->cuda_index ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaSetDevice ", status, {return NULL;} ); /* Allocate memory on it, for one tile */ status = (cudaError_t)cudaMalloc( &gpu_copy->device_private, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMalloc ", status, {return NULL;} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; + /* + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = memIndex + };*/ + /* Allocate memory on it, for one tile */ + ze_result_t status = zeMemAllocDevice(level_zero_device->ze_context, NULL, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, + level_zero_device->ze_device, &gpu_copy->device_private); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); +#else + gpu_copy->device_private = malloc(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE)); +#endif /* Attach this copy to the data, on the corresponding device */ - parsec_data_copy_attach(dta, gpu_copy, cuda_device->super.super.device_index); + parsec_data_copy_attach(dta, gpu_copy, device->device_index); /* We also need to tell PaRSEC that the owner of this data is the GPU, or the * GPU might not be selected to work on that data */ - parsec_data_transfer_ownership_to_copy(dta, cuda_device->super.super.device_index, PARSEC_FLOW_ACCESS_RW); + parsec_data_transfer_ownership_to_copy(dta, device->device_index, PARSEC_FLOW_ACCESS_RW); +#if defined(PARSEC_HAVE_CUDA) /* And copy the tile from CPU to GPU */ status = (cudaError_t)cudaMemcpy( gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, cudaMemcpyHostToDevice ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMemcpy ", status, {return NULL;} ); +#elif defined(PARSEC_HAVE_LEVEL_ZERO) + parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; + ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; + status = zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); + while(1) { + status = zeEventQueryStatus(copySignalEvent); + if(status == ZE_RESULT_SUCCESS) + break; + if(status != ZE_RESULT_NOT_READY) + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventHostSynchronize ", status, { break; } ); + usleep(1000); + } +#else + memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); +#endif g++; } } From 22d8bb921558ba6cfd51be35978b07f4689d42fb Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Fri, 14 Oct 2022 13:53:38 -0400 Subject: [PATCH 05/15] Try to automate DPCPP generated code compilation; fix ordinal of memory allocation request in wrapper. --- cmake_modules/ParsecCompilePTG.cmake | 35 +++++++++++++-- parsec/interfaces/ptg/ptg-compiler/jdf.h | 2 +- parsec/interfaces/ptg/ptg-compiler/jdf2c.c | 4 +- parsec/interfaces/ptg/ptg-compiler/main.c | 18 ++++---- tests/dsl/ptg/cuda/CMakeLists.txt | 2 +- tests/dsl/ptg/cuda/nvlink_wrapper.c | 51 ++++++++++++---------- 6 files changed, 72 insertions(+), 40 deletions(-) diff --git a/cmake_modules/ParsecCompilePTG.cmake b/cmake_modules/ParsecCompilePTG.cmake index 758ea8874..a7723d8da 100644 --- a/cmake_modules/ParsecCompilePTG.cmake +++ b/cmake_modules/ParsecCompilePTG.cmake @@ -7,7 +7,7 @@ # function(target_ptg_source_ex) set(options DEBUG LINE FORCE_PROFILE) - set(oneValueArgs TARGET MODE SOURCE DESTINATION DESTINATION_C DESTINATION_H FUNCTION_NAME DEP_MANAGEMENT) + set(oneValueArgs TARGET MODE SOURCE DESTINATION DESTINATION_C DESTINATION_H DESTINATION_DPCPP FUNCTION_NAME DEP_MANAGEMENT) set(multipleValueArgs WARNINGS IGNORE_PROPERTIES PTGPP_FLAGS) cmake_parse_arguments(PARSEC_PTGPP "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -47,6 +47,11 @@ function(target_ptg_source_ex) else() set(outname_h "${outname}.h") endif() + if(DEFINED PARSEC_PTGPP_DESTINATION_DPGPP) + set(outname_dpcpp "${PARSEC_PTGPP_DESTINATION_DPGPP}") + else() + set(outname_dpcpp "${outname}.dpcpp.C") + endif() if(DEFINED PARSEC_PTGPP_FUNCTION_NAME) set(fnname "${PARSEC_PTGPP_FUNCTION_NAME}") @@ -84,12 +89,21 @@ function(target_ptg_source_ex) # its cmake source_file name, yet we depend on the source_file name as it is how cmake tracks it get_property(location SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY LOCATION) - add_custom_command( + if(PARSEC_HAVE_LEVEL_ZERO) + add_custom_command( + OUTPUT ${outname_h} ${outname_c} ${outname_dpcpp} + COMMAND $ ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -D ${outname_dpcpp} -f ${fnname} + MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE} + DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp) + add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c} ${outname_dpcpp}) + else(PARSEC_HAVE_LEVEL_ZERO) + add_custom_command( OUTPUT ${outname_h} ${outname_c} COMMAND $ ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -f ${fnname} MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE} DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp) - add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c}) + add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c}) + endif(PARSEC_HAVE_LEVEL_ZERO) # Copy the properties to the generated files get_property(cflags SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY COMPILE_OPTIONS) @@ -103,11 +117,24 @@ function(target_ptg_source_ex) COMPILE_OPTIONS "${cflags}" INCLUDE_DIRECTORIES "${includes}" COMPILE_DEFINITIONS "${defs}") + if(PARSEC_HAVE_LEVEL_ZERO) + set_source_files_properties("${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}" + TARGET_DIRECTORY ${target} + PROPERTIES + GENERATED 1 + COMPILE_OPTIONS "${cflags}" + INCLUDE_DIRECTORIES "${includes}" + COMPILE_DEFINITIONS "${defs}") + endif(PARSEC_HAVE_LEVEL_ZERO) # make sure we produce .h before we build other .c in the target add_dependencies(${target} ptgpp_${target}.${outname}) # add to the target - target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}") + if(PARSEC_HAVE_LEVEL_ZERO) + target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c};${CMAKE_CURRENT_BINARY_DIR}/${outname_dpgpp}") + else(PARSEC_HAVE_LEVEL_ZERO) + target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}") + endif(PARSEC_HAVE_LEVEL_ZERO) get_target_property(_includes ${target} INCLUDE_DIRECTORIES) list(FIND _includes "${CMAKE_CURRENT_BINARY_DIR}" _i1) diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf.h b/parsec/interfaces/ptg/ptg-compiler/jdf.h index 0d9f268ce..81e9c7111 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf.h +++ b/parsec/interfaces/ptg/ptg-compiler/jdf.h @@ -96,7 +96,7 @@ typedef struct jdf_compiler_global_args { char *input; char *output_c; char *output_h; - char *output_driver_basename; + char *output_dpcpp; char *output_o; char *funcid; jdf_warning_mask_t wmask; diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index 37b59f268..a72ab3c0c 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -8777,13 +8777,13 @@ int jdf_optimize( jdf_t* jdf ) #endif int jdf2c(const char *output_c, const char *output_h, - const char *driver_output_basename, const char *_jdf_basename, jdf_t *jdf) + const char *dpcpp_output_basename, const char *_jdf_basename, jdf_t *jdf) { int ret = 0; jdf_cfilename = output_c; jdf_hfilename = strdup(output_h); - asprintf(&jdf_dpcppfilename, "%s.dpcpp.C", driver_output_basename); + asprintf(&jdf_dpcppfilename, "%s.C", dpcpp_output_basename); jdf_basename = _jdf_basename; cfile = NULL; hfile = NULL; diff --git a/parsec/interfaces/ptg/ptg-compiler/main.c b/parsec/interfaces/ptg/ptg-compiler/main.c index 1826ee850..b8e5e38f1 100644 --- a/parsec/interfaces/ptg/ptg-compiler/main.c +++ b/parsec/interfaces/ptg/ptg-compiler/main.c @@ -29,7 +29,7 @@ static jdf_compiler_global_args_t DEFAULTS = { .input = "-", .output_c = "a.c", .output_h = "a.h", - .output_driver_basename = "a", + .output_dpcpp = "a.dpcpp", .output_o = "a.o", .funcid = "a", .wmask = JDF_ALL_WARNINGS, @@ -60,9 +60,7 @@ static void usage(void) " --output-c, --output-h, and --function-name\n" " --output-c|-C Set the name of the .c output file (default '%s' or BASE.c)\n" " --output-h|-H Set the name of the .h output file (default '%s' or BASE.h)\n" - " --output-driver Set the base name for additional driver files (e.g. DPC++ bodies).\n" - " (default '%s' or BASE). Changing this value has precedence over\n" - " the defaults of --output\n" + " --output-dpcpp|-D Set the based name of the .dpcpp.* output files (default '%s' or BASE.dpcpp)\n" " --function-name|-f Set the unique identifier of the generated function\n" " The generated function will be called PaRSEC__new\n" " (default %s)\n" @@ -93,7 +91,7 @@ static void usage(void) DEFAULTS.input, DEFAULTS.output_c, DEFAULTS.output_h, - DEFAULTS.output_driver_basename, + DEFAULTS.output_dpcpp, DEFAULTS.funcid, (DEFAULTS.dep_management == DEP_MANAGEMENT_INDEX_ARRAY ? DEP_MANAGEMENT_INDEX_ARRAY_STRING : (DEFAULTS.dep_management == DEP_MANAGEMENT_DYNAMIC_HASH_TABLE ? DEP_MANAGEMENT_DYNAMIC_HASH_TABLE_STRING : @@ -178,7 +176,7 @@ static void parse_args(int argc, char *argv[]) { "output-c", required_argument, NULL, 'C' }, { "output-h", required_argument, NULL, 'H' }, { "output-o", required_argument, NULL, 'O' }, - { "output-driver", required_argument, NULL, 'D' }, + { "output-dpcpp", required_argument, NULL, 'D' }, { "output", required_argument, NULL, 'o' }, { "function-name", required_argument, NULL, 'f' }, { "Wmasked", no_argument, &wmasked, 1 }, @@ -319,12 +317,12 @@ static void parse_args(int argc, char *argv[]) } if( NULL != d ) { - JDF_COMPILER_GLOBAL_ARGS.output_driver_basename = d; + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = d; } else { if(NULL != o) { - JDF_COMPILER_GLOBAL_ARGS.output_driver_basename = strdup(o); + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = strdup(o); } else - JDF_COMPILER_GLOBAL_ARGS.output_driver_basename = DEFAULTS.output_driver_basename; + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = DEFAULTS.output_dpcpp; } if( NULL == c) { @@ -436,7 +434,7 @@ int main(int argc, char *argv[]) if( jdf2c(JDF_COMPILER_GLOBAL_ARGS.output_c, JDF_COMPILER_GLOBAL_ARGS.output_h, - JDF_COMPILER_GLOBAL_ARGS.output_driver_basename, + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp, JDF_COMPILER_GLOBAL_ARGS.funcid, ¤t_jdf) < 0 ) { return 1; diff --git a/tests/dsl/ptg/cuda/CMakeLists.txt b/tests/dsl/ptg/cuda/CMakeLists.txt index 6ce518e9f..cf1761f1f 100644 --- a/tests/dsl/ptg/cuda/CMakeLists.txt +++ b/tests/dsl/ptg/cuda/CMakeLists.txt @@ -43,7 +43,7 @@ if(PARSEC_HAVE_LEVEL_ZERO) parsec_addtest_executable(C nvlink SOURCES nvlink_main.c nvlink_wrapper.c) target_include_directories(nvlink PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) target_ptg_sources(nvlink PRIVATE "nvlink.jdf") - target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP) + target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) else(TARGET MKL::MKL_DPCPP) message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink test with DPCPP is disabled") endif(TARGET MKL::MKL_DPCPP) diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index a5ae5324d..a995316de 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -205,27 +205,34 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb parsec_data_copy_t *gpu_copy = PARSEC_OBJ_NEW(parsec_data_copy_t); #if defined(PARSEC_HAVE_CUDA) /* We chose the GPU */ - parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t *)device; + parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t *)device; cudaError_t status = cudaSetDevice( cuda_device->cuda_index ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaSetDevice ", status, {return NULL;} ); /* Allocate memory on it, for one tile */ status = (cudaError_t)cudaMalloc( &gpu_copy->device_private, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMalloc ", status, {return NULL;} ); #elif defined(PARSEC_HAVE_LEVEL_ZERO) - parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; - /* - ze_device_mem_alloc_desc_t memAllocDesc = { + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; + ze_device_memory_properties_t devMemProperties; + int count = 1; + /* Safety: we check that there is one memory segment available on this device (ordinal 0), and + * that this segment is big enough to store what we need to allocate */ + status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, &devMemProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { free(devMemProperties); return NULL; }); + assert(count >= 1); + assert(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) + 128 <= devMemProperties.totalSize); + ze_device_mem_alloc_desc_t memAllocDesc = { .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, .pNext = NULL, .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, - .ordinal = memIndex - };*/ - /* Allocate memory on it, for one tile */ - ze_result_t status = zeMemAllocDevice(level_zero_device->ze_context, NULL, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, + .ordinal = 0 + }; + /* Allocate memory on it, for one tile */ + ze_result_t status = zeMemAllocDevice(level_zero_device->ze_context, NULL, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, level_zero_device->ze_device, &gpu_copy->device_private); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); #else - gpu_copy->device_private = malloc(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE)); + gpu_copy->device_private = malloc(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE)); #endif /* Attach this copy to the data, on the corresponding device */ parsec_data_copy_attach(dta, gpu_copy, device->device_index); @@ -240,20 +247,20 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb cudaMemcpyHostToDevice ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMemcpy ", status, {return NULL;} ); #elif defined(PARSEC_HAVE_LEVEL_ZERO) - parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; - ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; - status = zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); - while(1) { - status = zeEventQueryStatus(copySignalEvent); - if(status == ZE_RESULT_SUCCESS) - break; - if(status != ZE_RESULT_NOT_READY) - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventHostSynchronize ", status, { break; } ); - usleep(1000); - } + parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; + ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; + status = zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); + while(1) { + status = zeEventQueryStatus(copySignalEvent); + if(status == ZE_RESULT_SUCCESS) + break; + if(status != ZE_RESULT_NOT_READY) + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventHostSynchronize ", status, { break; } ); + usleep(1000); + } #else - memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); + memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); #endif g++; } From 84fe4b1450c286182416701b0399561ebab69d7c Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Sun, 16 Oct 2022 13:06:47 -0400 Subject: [PATCH 06/15] Command Lists need to be sent to the Command Queue if they are not created immediate (and they cannot be immediate if we want to get their Command Queue, which is necessary for the DPC++ interface) Typo and multiple CMake fixes to make CMake link with DPCPP generated files --- CMakeLists.txt | 3 +++ cmake_modules/ParsecCompilePTG.cmake | 24 +++++++++---------- parsec/interfaces/ptg/ptg-compiler/jdf2c.c | 4 ++-- parsec/interfaces/ptg/ptg-compiler/jdf2c.h | 2 +- parsec/interfaces/ptg/ptg-compiler/main.c | 4 ++-- .../level_zero/device_level_zero_module.c | 6 +++++ tests/dsl/ptg/cuda/CMakeLists.txt | 2 +- tests/dsl/ptg/cuda/nvlink_wrapper.c | 18 +++++++------- 8 files changed, 35 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c8fb925b8..a7047cfab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,9 @@ include(CMakeDependentOption) include(CMakePushCheckState) include(GNUInstallDirs) +# Help debug dependencies with custom targets that PTG uses a lot +SET(GRAPHVIZ_CUSTOM_TARGETS TRUE) + # The current version number # This uses the numbering scheme from libtool -version_number c:r:a # see http://www.sourceware.org/autobook/autobook/autobook_61.html diff --git a/cmake_modules/ParsecCompilePTG.cmake b/cmake_modules/ParsecCompilePTG.cmake index a7723d8da..9ba511205 100644 --- a/cmake_modules/ParsecCompilePTG.cmake +++ b/cmake_modules/ParsecCompilePTG.cmake @@ -47,10 +47,8 @@ function(target_ptg_source_ex) else() set(outname_h "${outname}.h") endif() - if(DEFINED PARSEC_PTGPP_DESTINATION_DPGPP) - set(outname_dpcpp "${PARSEC_PTGPP_DESTINATION_DPGPP}") - else() - set(outname_dpcpp "${outname}.dpcpp.C") + if(DEFINED PARSEC_PTGPP_DESTINATION_DPCPP) + set(outname_dpcpp "${PARSEC_PTGPP_DESTINATION_DPCPP}") endif() if(DEFINED PARSEC_PTGPP_FUNCTION_NAME) @@ -89,21 +87,21 @@ function(target_ptg_source_ex) # its cmake source_file name, yet we depend on the source_file name as it is how cmake tracks it get_property(location SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY LOCATION) - if(PARSEC_HAVE_LEVEL_ZERO) + if(DEFINED outname_dpcpp) add_custom_command( OUTPUT ${outname_h} ${outname_c} ${outname_dpcpp} COMMAND $ ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -D ${outname_dpcpp} -f ${fnname} MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE} DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp) add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c} ${outname_dpcpp}) - else(PARSEC_HAVE_LEVEL_ZERO) + else(DEFINED outname_dpcpp) add_custom_command( OUTPUT ${outname_h} ${outname_c} COMMAND $ ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -f ${fnname} MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE} DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp) add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c}) - endif(PARSEC_HAVE_LEVEL_ZERO) + endif(DEFINED outname_dpcpp) # Copy the properties to the generated files get_property(cflags SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY COMPILE_OPTIONS) @@ -117,7 +115,7 @@ function(target_ptg_source_ex) COMPILE_OPTIONS "${cflags}" INCLUDE_DIRECTORIES "${includes}" COMPILE_DEFINITIONS "${defs}") - if(PARSEC_HAVE_LEVEL_ZERO) + if(DEFINED outname_dpcpp) set_source_files_properties("${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}" TARGET_DIRECTORY ${target} PROPERTIES @@ -125,16 +123,16 @@ function(target_ptg_source_ex) COMPILE_OPTIONS "${cflags}" INCLUDE_DIRECTORIES "${includes}" COMPILE_DEFINITIONS "${defs}") - endif(PARSEC_HAVE_LEVEL_ZERO) + endif(DEFINED outname_dpcpp) # make sure we produce .h before we build other .c in the target add_dependencies(${target} ptgpp_${target}.${outname}) # add to the target - if(PARSEC_HAVE_LEVEL_ZERO) - target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c};${CMAKE_CURRENT_BINARY_DIR}/${outname_dpgpp}") - else(PARSEC_HAVE_LEVEL_ZERO) + if(DEFINED outname_dpcpp) + target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c};${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}") + else(DEFINED outname_dpcpp) target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}") - endif(PARSEC_HAVE_LEVEL_ZERO) + endif(DEFINED outname_dpcpp) get_target_property(_includes ${target} INCLUDE_DIRECTORIES) list(FIND _includes "${CMAKE_CURRENT_BINARY_DIR}" _i1) diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index a72ab3c0c..e50b8924d 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -8777,13 +8777,13 @@ int jdf_optimize( jdf_t* jdf ) #endif int jdf2c(const char *output_c, const char *output_h, - const char *dpcpp_output_basename, const char *_jdf_basename, jdf_t *jdf) + const char *output_dpcpp, const char *_jdf_basename, jdf_t *jdf) { int ret = 0; jdf_cfilename = output_c; jdf_hfilename = strdup(output_h); - asprintf(&jdf_dpcppfilename, "%s.C", dpcpp_output_basename); + jdf_dpcppfilename = strdup(output_dpcpp); jdf_basename = _jdf_basename; cfile = NULL; hfile = NULL; diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.h b/parsec/interfaces/ptg/ptg-compiler/jdf2c.h index 5c36913c8..e308acdd3 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.h +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.h @@ -11,6 +11,6 @@ int jdf_optimize( jdf_t* jdf ); int jdf2c(const char *output_c, const char *output_h, - const char *driver_basename, const char *_basename, jdf_t *jdf); + const char *output_dpcpp, const char *_basename, jdf_t *jdf); #endif /* _jdf2c_h */ diff --git a/parsec/interfaces/ptg/ptg-compiler/main.c b/parsec/interfaces/ptg/ptg-compiler/main.c index b8e5e38f1..ee8ac6ae2 100644 --- a/parsec/interfaces/ptg/ptg-compiler/main.c +++ b/parsec/interfaces/ptg/ptg-compiler/main.c @@ -29,7 +29,7 @@ static jdf_compiler_global_args_t DEFAULTS = { .input = "-", .output_c = "a.c", .output_h = "a.h", - .output_dpcpp = "a.dpcpp", + .output_dpcpp = "a", .output_o = "a.o", .funcid = "a", .wmask = JDF_ALL_WARNINGS, @@ -317,7 +317,7 @@ static void parse_args(int argc, char *argv[]) } if( NULL != d ) { - JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = d; + JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = strdup(d); } else { if(NULL != o) { JDF_COMPILER_GLOBAL_ARGS.output_dpcpp = strdup(o); diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c index 68547513c..0ce82df5f 100644 --- a/parsec/mca/device/level_zero/device_level_zero_module.c +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -2013,6 +2013,12 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, ze_result_t ze_rc = zeCommandListClose(level_zero_stream->level_zero_cl); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { } ); + ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero-stream->level_zero_cl, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, + { } ); + ze_rc = zeCommandListReset(level_zero-stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, + { } ); } stream->tasks[stream->start] = task; stream->start = (stream->start + 1) % stream->max_events; diff --git a/tests/dsl/ptg/cuda/CMakeLists.txt b/tests/dsl/ptg/cuda/CMakeLists.txt index cf1761f1f..184efaabc 100644 --- a/tests/dsl/ptg/cuda/CMakeLists.txt +++ b/tests/dsl/ptg/cuda/CMakeLists.txt @@ -42,7 +42,7 @@ if(PARSEC_HAVE_LEVEL_ZERO) parsec_addtest_executable(C nvlink SOURCES nvlink_main.c nvlink_wrapper.c) target_include_directories(nvlink PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) - target_ptg_sources(nvlink PRIVATE "nvlink.jdf") + target_ptg_source_ex(SOURCE "nvlink.jdf" MODE PRIVATE TARGET nvlink DESTINATION_DPCPP "nvlink.dpcpp.C") target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) else(TARGET MKL::MKL_DPCPP) message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink test with DPCPP is disabled") diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index a995316de..2e52cab22 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -214,25 +214,25 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb #elif defined(PARSEC_HAVE_LEVEL_ZERO) parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; ze_device_memory_properties_t devMemProperties; - int count = 1; + uint32_t count = 1; /* Safety: we check that there is one memory segment available on this device (ordinal 0), and * that this segment is big enough to store what we need to allocate */ - status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, &devMemProperties); - PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { free(devMemProperties); return NULL; }); + ze_result_t status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, &devMemProperties); + PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { return NULL; }); assert(count >= 1); assert(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) + 128 <= devMemProperties.totalSize); - ze_device_mem_alloc_desc_t memAllocDesc = { + ze_device_mem_alloc_desc_t memAllocDesc = { .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, .pNext = NULL, .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, .ordinal = 0 }; - /* Allocate memory on it, for one tile */ - ze_result_t status = zeMemAllocDevice(level_zero_device->ze_context, NULL, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, - level_zero_device->ze_device, &gpu_copy->device_private); + /* Allocate memory on it, for one tile */ + status = zeMemAllocDevice(level_zero_device->ze_context, &memAllocDesc, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, + level_zero_device->ze_device, &gpu_copy->device_private); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); #else - gpu_copy->device_private = malloc(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE)); + gpu_copy->device_private = malloc(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE)); #endif /* Attach this copy to the data, on the corresponding device */ parsec_data_copy_attach(dta, gpu_copy, device->device_index); @@ -247,7 +247,7 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb cudaMemcpyHostToDevice ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMemcpy ", status, {return NULL;} ); #elif defined(PARSEC_HAVE_LEVEL_ZERO) - parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; + parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; status = zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); From c16acfd5047f0bea76714ebc6380b3aa832deb36 Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Wed, 19 Oct 2022 14:05:31 -0400 Subject: [PATCH 07/15] Add a standalone test for Zero Level capability and integration with DPC++ kernels --- cmake_modules/Findlevel-zero.cmake | 34 +- parsec/mca/device/device.c | 2 +- .../level_zero/device_level_zero_module.c | 4 +- tests/CMakeLists.txt | 1 + tests/dsl/ptg/cuda/CMakeLists.txt | 41 +- tests/dsl/ptg/cuda/nvlink.jdf | 50 ++- tests/dsl/ptg/cuda/nvlink_wrapper.c | 42 +- tests/subsystem/CMakeLists.txt | 5 + tests/subsystem/level_zero/CMakeLists.txt | 14 + tests/subsystem/level_zero/interface.dpcpp.C | 26 ++ tests/subsystem/level_zero/interface.dpcpp.h | 42 ++ tests/subsystem/level_zero/kernel.dpcpp.C | 53 +++ tests/subsystem/level_zero/main.c | 386 ++++++++++++++++++ 13 files changed, 637 insertions(+), 63 deletions(-) create mode 100644 tests/subsystem/CMakeLists.txt create mode 100644 tests/subsystem/level_zero/CMakeLists.txt create mode 100644 tests/subsystem/level_zero/interface.dpcpp.C create mode 100644 tests/subsystem/level_zero/interface.dpcpp.h create mode 100644 tests/subsystem/level_zero/kernel.dpcpp.C create mode 100644 tests/subsystem/level_zero/main.c diff --git a/cmake_modules/Findlevel-zero.cmake b/cmake_modules/Findlevel-zero.cmake index 4694dcb42..51b63912e 100644 --- a/cmake_modules/Findlevel-zero.cmake +++ b/cmake_modules/Findlevel-zero.cmake @@ -4,28 +4,28 @@ if(LEVEL_ZERO_ROOT_DIR) find_path (LEVEL_ZERO_INCLUDE_DIR NAMES "level_zero/ze_api.h" PATHS "${LEVEL_ZERO_ROOT_DIR}/include" NO_DEFAULT_PATH) if(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) - get_filename_component(ZE_LOADER_LIBRARY_DIR ${ZE_LOADER_LIBRARY} DIRECTORY) - check_library_exists("ze_loader" "zeInit" ${ZE_LOADER_LIBRARY_DIR} ZE_LOADER_HAVE_ZEINIT) + get_filename_component(ZE_LOADER_LIBRARY_DIR ${ZE_LOADER_LIBRARY} DIRECTORY) + check_library_exists("ze_loader" "zeInit" ${ZE_LOADER_LIBRARY_DIR} ZE_LOADER_HAVE_ZEINIT) - if(ZE_LOADER_HAVE_ZEINIT) - message(STATUS "Found ze_loader library in ${ZE_LOADER_LIBRARY} and level_zero/ze_api.h in ${LEVEL_ZERO_INCLUDE_DIR}") - add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) - set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${ZE_LOADER_LIBRARY}") + if(ZE_LOADER_HAVE_ZEINIT) + message(STATUS "Found ze_loader library in ${ZE_LOADER_LIBRARY} and level_zero/ze_api.h in ${LEVEL_ZERO_INCLUDE_DIR}") + add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL) + set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${ZE_LOADER_LIBRARY}") set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h") include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") - set(LEVEL_ZERO_FOUND TRUE) - else(ZE_LOADER_HAVE_ZEINIT) - if(NOT ZE_LOADER_HAVE_ZEINIT) + set(LEVEL_ZERO_FOUND TRUE) + else(ZE_LOADER_HAVE_ZEINIT) + if(NOT ZE_LOADER_HAVE_ZEINIT) message(WARNING "Found ze_loader library under ${ZE_LOADER_LIBRARY}, but could not find symbol zeInit in this library -- falling back to package config search") - endif(NOT ZE_LOADER_HAVE_ZEINIT) - endif(ZE_LOADER_HAVE_ZEINIT) + endif(NOT ZE_LOADER_HAVE_ZEINIT) + endif(ZE_LOADER_HAVE_ZEINIT) else(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) - if(NOT ZE_LOADER_LIBRARY) - message(WARNING "Could not find ze_loader library under provided LEVEL_ZERO_ROOT_DIR='${LEVEL_ZERO_ROOT_DIR}' (tried subdirectories lib/ and lib64/) -- falling back to package config search") - endif(NOT ZE_LOADER_LIBRARY) - if(NOT LEVEL_ZERO_INCLUDE_DIR) - message(WARNING "Cound not find level_zero/ze_api.h under provided LEVEL_ZERO_ROOT_DIR=${LEVEL_ZERO_ROOT_DIR}' (tried subdirectory include) -- falling back to package config search") - endif(NOT LEVEL_ZERO_INCLUDE_DIR) + if(NOT ZE_LOADER_LIBRARY) + message(WARNING "Could not find ze_loader library under provided LEVEL_ZERO_ROOT_DIR='${LEVEL_ZERO_ROOT_DIR}' (tried subdirectories lib/ and lib64/) -- falling back to package config search") + endif(NOT ZE_LOADER_LIBRARY) + if(NOT LEVEL_ZERO_INCLUDE_DIR) + message(WARNING "Cound not find level_zero/ze_api.h under provided LEVEL_ZERO_ROOT_DIR=${LEVEL_ZERO_ROOT_DIR}' (tried subdirectory include) -- falling back to package config search") + endif(NOT LEVEL_ZERO_INCLUDE_DIR) endif(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR) endif(LEVEL_ZERO_ROOT_DIR) diff --git a/parsec/mca/device/device.c b/parsec/mca/device/device.c index a8d7e326d..57f147427 100644 --- a/parsec/mca/device/device.c +++ b/parsec/mca/device/device.c @@ -88,7 +88,7 @@ int parsec_get_best_device( parsec_task_t* this_task, double ratio ) /* Make sure data_in is not NULL */ if( NULL == this_task->data[i].data_in ) continue; /* And that we have a data (aka it is not NEW) */ - if( NULL == this_task->data[i].source_repo_entry ) continue; + if( 0 && NULL == this_task->data[i].source_repo_entry ) continue; /* Data is updated by the task, and we try to minimize the data movements */ if( (NULL != this_task->task_class->out[i]) && diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c index 0ce82df5f..0bdf9869e 100644 --- a/parsec/mca/device/level_zero/device_level_zero_module.c +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -2013,10 +2013,10 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, ze_result_t ze_rc = zeCommandListClose(level_zero_stream->level_zero_cl); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { } ); - ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero-stream->level_zero_cl, NULL); + ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->level_zero_cl, NULL); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { } ); - ze_rc = zeCommandListReset(level_zero-stream->level_zero_cl); + ze_rc = zeCommandListReset(level_zero_stream->level_zero_cl); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { } ); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f0babe622..81d6add27 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -68,6 +68,7 @@ target_link_libraries(tests_common PRIVATE parsec) add_subdirectory(class) add_subdirectory(api) +add_subdirectory(subsystem) if( TARGET parsec-ptgpp ) add_subdirectory(runtime) add_subdirectory(dsl/ptg) diff --git a/tests/dsl/ptg/cuda/CMakeLists.txt b/tests/dsl/ptg/cuda/CMakeLists.txt index 184efaabc..7904e5671 100644 --- a/tests/dsl/ptg/cuda/CMakeLists.txt +++ b/tests/dsl/ptg/cuda/CMakeLists.txt @@ -1,4 +1,22 @@ -if(PARSEC_HAVE_CUDA) +if(PARSEC_HAVE_LEVEL_ZERO) + if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) +if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") +endif(NOT MKL_FOUND) + endif(NOT TARGET MKL::MKL_DPCPP) + if(TARGET MKL::MKL_DPCPP) +message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + include(ParsecCompilePTG) + + parsec_addtest_executable(C nvlink SOURCES nvlink_main.c nvlink_wrapper.c) + target_include_directories(nvlink PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) +target_ptg_source_ex(SOURCE "nvlink.jdf" MODE PRIVATE TARGET nvlink DESTINATION_DPCPP "nvlink.dpcpp.C") + target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) + else(TARGET MKL::MKL_DPCPP) +message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink_lz test with DPCPP is disabled") + endif(TARGET MKL::MKL_DPCPP) +elseif(PARSEC_HAVE_CUDA) if( NOT TARGET CUDA::cublas ) message(WARNING "CUBLAS library not automatically detected when CUDA was found -- not enabling nvlink test") else( NOT TARGET CUDA::cublas ) @@ -27,24 +45,5 @@ if(PARSEC_HAVE_CUDA) target_ptg_sources(testing_get_best_device PRIVATE "get_best_device_check.jdf") parsec_addtest_cmd(get_best_device_shm ${SHM_TEST_CMD_LIST} ./testing_get_best_device -N 400 -t 20 -g 1) -endif(PARSEC_HAVE_CUDA) - -if(PARSEC_HAVE_LEVEL_ZERO) - if(NOT TARGET MKL::MKL_DPCPP) - find_package(MKL) - if(NOT MKL_FOUND) - message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") - endif(NOT MKL_FOUND) - endif(NOT TARGET MKL::MKL_DPCPP) - if(TARGET MKL::MKL_DPCPP) - message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") - include(ParsecCompilePTG) - - parsec_addtest_executable(C nvlink SOURCES nvlink_main.c nvlink_wrapper.c) - target_include_directories(nvlink PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) - target_ptg_source_ex(SOURCE "nvlink.jdf" MODE PRIVATE TARGET nvlink DESTINATION_DPCPP "nvlink.dpcpp.C") - target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) - else(TARGET MKL::MKL_DPCPP) - message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink test with DPCPP is disabled") - endif(TARGET MKL::MKL_DPCPP) endif(PARSEC_HAVE_LEVEL_ZERO) + diff --git a/tests/dsl/ptg/cuda/nvlink.jdf b/tests/dsl/ptg/cuda/nvlink.jdf index 70397a934..8ce2f6ebb 100644 --- a/tests/dsl/ptg/cuda/nvlink.jdf +++ b/tests/dsl/ptg/cuda/nvlink.jdf @@ -162,11 +162,28 @@ BODY [type=DPCPP { double alpha=0.0; double beta=1.0; - oneapi::mkl::blas::gemm(parsec_dpcpp->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + try { + oneapi::mkl::blas::gemm(parsec_dpcpp->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, descA->super.mb, descA->super.nb, descA->super.mb, alpha, static_cast(A), descA->super.mb, static_cast(A), descA->super.mb, beta, static_cast(C), descA->super.mb); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } + fprintf(stderr, "GEMM1(%d, %d, %d) has been scheduled on OneAPI MKL BLAS using the DPC++ driver\n", m, g, r); } END @@ -215,6 +232,37 @@ BODY [type=CUDA } END +BODY [type=DPCPP + weight=(1)] +{ + double alpha=0.0; + double beta=1.0; + try { + oneapi::mkl::blas::gemm(parsec_dpcpp->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + descA->super.mb, descA->super.nb, descA->super.mb, + alpha, static_cast(A), descA->super.mb, + static_cast(A), descA->super.mb, + beta, static_cast(C), descA->super.mb); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM2 throws unexpected exception that is also badly formatted..."); + } + fprintf(stderr, "GEMM2(%d, %d, %d) has been scheduled on OneAPI MKL BLAS using the DPC++ driver\n", m, g, r); +} +END + + BODY { fprintf(stderr, "Kernel GEMM2(%d, %d, %d) in nvlink test is running on a CPU, which is not the intended behavior\n", diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index 2e52cab22..269df7ba9 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -89,11 +89,11 @@ __parsec_nvlink_destructor( parsec_nvlink_taskpool_t* nvlink_taskpool) status = (cudaError_t)cudaFree( gpu_copy->device_private ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaFree ", status, {} ); #elif defined(PARSEC_HAVE_LEVEL_ZERO) - parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; + parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; ze_result_t status = zeMemFree(level_zero_device->ze_context, gpu_copy->device_private); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, {} ); #else - free(gpu_copy->device_private); + free(gpu_copy->device_private); #endif gpu_copy->device_private = NULL; parsec_data_copy_detach(dta, gpu_copy, device->device_index); @@ -204,8 +204,8 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb /* And we create a new data copy on GPU */ parsec_data_copy_t *gpu_copy = PARSEC_OBJ_NEW(parsec_data_copy_t); #if defined(PARSEC_HAVE_CUDA) - /* We chose the GPU */ - parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t *)device; + /* We chose the GPU */ + parsec_device_cuda_module_t *cuda_device = (parsec_device_cuda_module_t *)device; cudaError_t status = cudaSetDevice( cuda_device->cuda_index ); PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaSetDevice ", status, {return NULL;} ); /* Allocate memory on it, for one tile */ @@ -213,7 +213,7 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMalloc ", status, {return NULL;} ); #elif defined(PARSEC_HAVE_LEVEL_ZERO) parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; - ze_device_memory_properties_t devMemProperties; + ze_device_memory_properties_t devMemProperties; uint32_t count = 1; /* Safety: we check that there is one memory segment available on this device (ordinal 0), and * that this segment is big enough to store what we need to allocate */ @@ -222,10 +222,10 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb assert(count >= 1); assert(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) + 128 <= devMemProperties.totalSize); ze_device_mem_alloc_desc_t memAllocDesc = { - .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, - .pNext = NULL, - .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, - .ordinal = 0 + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = 0 }; /* Allocate memory on it, for one tile */ status = zeMemAllocDevice(level_zero_device->ze_context, &memAllocDesc, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, @@ -248,19 +248,19 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMemcpy ", status, {return NULL;} ); #elif defined(PARSEC_HAVE_LEVEL_ZERO) parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; - ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; - status = zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); - while(1) { - status = zeEventQueryStatus(copySignalEvent); - if(status == ZE_RESULT_SUCCESS) - break; - if(status != ZE_RESULT_NOT_READY) - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventHostSynchronize ", status, { break; } ); - usleep(1000); - } + ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; + status = zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); + while(1) { + status = zeEventQueryStatus(copySignalEvent); + if(status == ZE_RESULT_SUCCESS) + break; + if(status != ZE_RESULT_NOT_READY) + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventHostSynchronize ", status, { break; } ); + usleep(1000); + } #else - memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); + memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); #endif g++; } diff --git a/tests/subsystem/CMakeLists.txt b/tests/subsystem/CMakeLists.txt new file mode 100644 index 000000000..863948e3b --- /dev/null +++ b/tests/subsystem/CMakeLists.txt @@ -0,0 +1,5 @@ +if(PARSEC_HAVE_LEVEL_ZERO) + add_subdirectory(level_zero) +else(PARSEC_HAVE_LEVEL_ZERO) + message(WARNING "PaRSEC doesn't have Level Zero support?") +endif(PARSEC_HAVE_LEVEL_ZERO) diff --git a/tests/subsystem/level_zero/CMakeLists.txt b/tests/subsystem/level_zero/CMakeLists.txt new file mode 100644 index 000000000..de11e7887 --- /dev/null +++ b/tests/subsystem/level_zero/CMakeLists.txt @@ -0,0 +1,14 @@ +if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) +endif(NOT TARGET MKL::MKL_DPCPP) +if(TARGET MKL::MKL_DPCPP) +message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + parsec_addtest_executable(C lz SOURCES main.c kernel.dpcpp.C interface.dpcpp.C) + target_include_directories(lz PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) + target_link_libraries(lz PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) +else(TARGET MKL::MKL_DPCPP) + message(STATUS "Target MKL::MKL_DPCPP is not defined, Level Zero test with DPCPP is disabled") +endif(TARGET MKL::MKL_DPCPP) diff --git a/tests/subsystem/level_zero/interface.dpcpp.C b/tests/subsystem/level_zero/interface.dpcpp.C new file mode 100644 index 000000000..46f802c69 --- /dev/null +++ b/tests/subsystem/level_zero/interface.dpcpp.C @@ -0,0 +1,26 @@ +#include "level_zero/ze_api.h" +#include "interface.dpcpp.h" + +sycl_wrapper_t *sycl_queue_create(ze_driver_handle_t ze_driver, + ze_device_handle_t ze_device, + ze_context_handle_t ze_context, + ze_command_queue_handle_t ze_queue) +{ + std::vectordevices; + + for(uint32_t i = 0; i < num_device; i++) { + devices.push_back(swd[i]->device); + } + sycl::backend_input_t hContextInteropInput = {ze_context, devices}; + res->context = sycl::make_context(hContextInteropInput); + res->queue = sycl::make_queue(ze_queue, res->context); + + return res; +} + +int sycl_queue_destroy(sycl_wrapper_t *sycl_obj) +{ + delete sycl_obj; + return 0; +} + diff --git a/tests/subsystem/level_zero/interface.dpcpp.h b/tests/subsystem/level_zero/interface.dpcpp.h new file mode 100644 index 000000000..79dc34f62 --- /dev/null +++ b/tests/subsystem/level_zero/interface.dpcpp.h @@ -0,0 +1,42 @@ +#ifndef INTERFACE_DPCPP_H +#define INTERFACE_DPCPP_H + +typedef struct sycl_wrapper_driver_s sycl_wrapper_driver_t; +typedef struct sycl_wrapper_device_s sycl_wrapper_device_t; +typedef struct sycl_wrapper_queue_s sycl_wrapper_queue_t; + +#if defined(c_plusplus) || defined(__cplusplus) +#include "sycl/ext/oneapi/backend/level_zero.hpp" + +struct sycl_wrapper_driver_s { + sycl::platform platform; + sycl::context context; +}; + +struct sycl_wrapper_device_s { + sycl::device device; +}; + +struct sycl_wrapper_queue_s { + sycl::queue queue; +}; + +extern "C" { +#endif + +sycl_wrapper_driver_t *sycl_wrapper_platform_create(ze_driver_handle_t ze_driver); +void sycl_wrapper_platform_add_context(sycl_wrapper_driver_t *swp, ze_context_handle_t ze_context, sycl_wrapper_device_t **swd, uint32_t num_device); +sycl_wrapper_device_t *sycl_wrapper_device_create(ze_device_handle_t ze_device); +sycl_wrapper_queue_t *sycl_wrapper_queue_create(sycl_wrapper_driver_t *swp, sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue); + +sycl_wrapper_t *sycl_queue_create(ze_driver_handle_t ze_driver, + ze_device_handle_t ze_device, + ze_context_handle_t ze_context, + ze_command_queue_handle_t ze_queue); +int sycl_queue_destroy(sycl_wrapper_t *_queue); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif //INTERFACE_DPCPP_H diff --git a/tests/subsystem/level_zero/kernel.dpcpp.C b/tests/subsystem/level_zero/kernel.dpcpp.C new file mode 100644 index 000000000..839f99627 --- /dev/null +++ b/tests/subsystem/level_zero/kernel.dpcpp.C @@ -0,0 +1,53 @@ +#include +#include "sycl/ext/oneapi/backend/level_zero.hpp" +#include "interface.dpcpp.h" + +extern "C" { + int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, + const double *A, + double *C, + int mb); +} + +int dpcpp_kernel_GEMM(void *_sw, + const double *A, + double *C, + int mb) +{ + sycl_wrapper_t *sw = reinterpret_cast(_sw); + + double alpha=0.0; + double beta=1.0; + sycl::backend_input_t> hBufferInteropInputA = { (void*)A, sycl::ext::oneapi::level_zero::ownership::keep }; + sycl::buffer bA = sycl::make_buffer(hBufferInteropInputA, swp->context); + auto bbA = bA.reinterpret(sycl::range<1>(mb*mb)); + sycl::backend_input_t> hBufferInteropInputC = { (void*)C, sycl::ext::oneapi::level_zero::ownership::keep }; + sycl::buffer bC = sycl::make_buffer(hBufferInteropInputC, swp->context); + auto bbC = bC.reinterpret(sycl::range<1>(mb*mb)); + try { + oneapi::mkl::blas::gemm(swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + mb, mb, mb, + alpha, bbA, mb, + bbA, mb, + beta, bbC, mb); + } catch (const oneapi::mkl::invalid_argument &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unimplemented exception"); + } catch (const std::exception& e) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception"); + } catch (...) { + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } + fprintf(stderr, "kernel has been scheduled on OneAPI MKL BLAS using the DPC++ driver\n"); + + return 0; +} diff --git a/tests/subsystem/level_zero/main.c b/tests/subsystem/level_zero/main.c new file mode 100644 index 000000000..6adb371ef --- /dev/null +++ b/tests/subsystem/level_zero/main.c @@ -0,0 +1,386 @@ +#include +#include + +#include +#include "interface.dpcpp.h" + +int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, + const double *A, + double *C, + int mb); + +struct driver_s; +struct device_s; +struct stream_s; + +#define NB_STREAMS 4 +#define MAX_EVENTS 2 +#define N 256 + +typedef struct stream_s { + int immediate; + void *sq; + ze_command_queue_handle_t cq; + ze_command_list_handle_t cl; + struct device_s *device; + ze_event_handle_t events[MAX_EVENTS]; +} stream_t; + +typedef struct device_s { + ze_device_handle_t device; + struct driver_s *driver; + sycl_wrapper_device_t *swd; + ze_event_pool_handle_t eventPool; + stream_t streams[NB_STREAMS]; +} device_t; + +typedef struct driver_s { + ze_driver_handle_t driver; + ze_context_handle_t context; + sycl_wrapper_driver_t *swp; + int nb_devices; + device_t *devices; +} driver_t; + +#define LEVEL_ZERO_CHECK_ERROR(STR, ERROR, CODE) \ + do { \ + if( ZE_RESULT_SUCCESS != (ERROR) ) { \ + fprintf(stderr, "%s:%d %s returns Error 0x%x", __FILE__, __LINE__,\ + (STR), (ERROR) ); \ + CODE; \ + } \ + } while(0) + +static int init_device(device_t *device, ze_device_handle_t gpuDevice) +{ + // Discover all command queue groups + uint32_t cmdqueueGroupCount = 0; + ze_result_t ze_rc; + + ze_rc = zeDeviceGetCommandQueueGroupProperties(gpuDevice, &cmdqueueGroupCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetCommandQueueGroupProperties (count) ", ze_rc, { return -1; } ); + + ze_command_queue_group_properties_t* cmdqueueGroupProperties = (ze_command_queue_group_properties_t*) + malloc(cmdqueueGroupCount * sizeof(ze_command_queue_group_properties_t)); + ze_rc = zeDeviceGetCommandQueueGroupProperties(gpuDevice, &cmdqueueGroupCount, cmdqueueGroupProperties); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetCommandQueueGroupProperties (populate) ", ze_rc, { return -1; } ); + + // Find a command queue type that support compute + //TODO: it might be more in line with the design to create different command queues for copy + // and compute than using the existing queues. + uint32_t computeQueueGroupOrdinal = cmdqueueGroupCount; + uint32_t copyQueueGroupOrdinal = cmdqueueGroupCount; + for( uint32_t i = 0; i < cmdqueueGroupCount && + (computeQueueGroupOrdinal == cmdqueueGroupCount || + copyQueueGroupOrdinal == cmdqueueGroupCount); ++i ) { + if( computeQueueGroupOrdinal == cmdqueueGroupCount && cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE ) { + computeQueueGroupOrdinal = i; + } + if( copyQueueGroupOrdinal == cmdqueueGroupCount && cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY ) { + copyQueueGroupOrdinal = i; + } + } + if( computeQueueGroupOrdinal == cmdqueueGroupCount ) { + fprintf(stderr, "level zero device: unable to find a Queue Group with COMPUTE flag"); + return -1; + } + if( copyQueueGroupOrdinal == cmdqueueGroupCount ) { + fprintf(stderr, "level zero device: unable to find a Queue Group with COMPUTE flag"); + return -1; + } + + // Create event pool + ze_event_pool_desc_t eventPoolDesc = { + ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, + NULL, + ZE_EVENT_POOL_FLAG_HOST_VISIBLE, // all events in pool are visible to Host + 1 // count + }; + ze_rc = zeEventPoolCreate(device->driver->context, &eventPoolDesc, 0, NULL, + &device->eventPool); + LEVEL_ZERO_CHECK_ERROR( "zeEventPoolCreate ", ze_rc, {return -1;} ); + + for(int j = 0; j < NB_STREAMS; j++ ) { + ze_command_queue_desc_t commandQueueDesc = { + ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, + NULL, + (uint32_t)-1, + 0, // index + 0, // flags + ZE_COMMAND_QUEUE_MODE_DEFAULT, + ZE_COMMAND_QUEUE_PRIORITY_NORMAL + }; + device->streams[j].device = device; + if( j < 2 ) { + device->streams[j].immediate = 1; + commandQueueDesc.ordinal = copyQueueGroupOrdinal; + ze_rc = zeCommandListCreateImmediate(device->driver->context, gpuDevice, + &commandQueueDesc, + &device->streams[j].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreateImmediate ", ze_rc, { return -1;} ); + } else { + device->streams[j].immediate = 0; + commandQueueDesc.ordinal = computeQueueGroupOrdinal; + ze_rc = zeCommandQueueCreate(device->driver->context, gpuDevice, + &commandQueueDesc, &device->streams[j].cq); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, { return -1;} ); + ze_command_list_desc_t commandListDesc = { + ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, + NULL, + computeQueueGroupOrdinal, + 0 // flags + }; + ze_rc = zeCommandListCreate(device->driver->context, gpuDevice, + &commandListDesc, &device->streams[j].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, { return -1;} ); + device->streams[j].sq = sycl_queue_create(device->driver->driver, gpuDevice, device->driver->context, device->streams[j].cq); + if(NULL == device->streams[j].sq) + return -1; + } + + for(int k = 0; k < MAX_EVENTS; k++ ) { + ze_event_desc_t eventDesc = { + ZE_STRUCTURE_TYPE_EVENT_DESC, + NULL, + 0, // index + 0, // no additional memory/cache coherency required on signal + ZE_EVENT_SCOPE_FLAG_HOST // ensure memory coherency across device and Host after event completes + }; + device->streams[j].events[k] = NULL; + ze_rc = zeEventCreate(device->eventPool, &eventDesc, &(device->streams[j].events[k])); + LEVEL_ZERO_CHECK_ERROR( "zeEventCreate ", ze_rc, {return -1;} ); + } + } + device->device = gpuDevice; + return 0; +} + +static int init_driver(driver_t *driver, int maxDevices) +{ + uint32_t deviceCount = 0; + ze_device_handle_t *allDevices; + ze_device_handle_t *gpuDevices; + ze_result_t ze_rc; + + ze_rc = zeDeviceGet(driver->driver, &deviceCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet (count) ", ze_rc, { return -1; } ); + + driver->nb_devices = 0; + driver->devices = NULL; + + if(deviceCount == 0) + return 0; + + allDevices = (ze_device_handle_t *)malloc(deviceCount * sizeof(ze_device_handle_t)); + gpuDevices = (ze_device_handle_t *)malloc(deviceCount * sizeof(ze_device_handle_t)); + ze_rc = zeDeviceGet(driver->driver, &deviceCount, allDevices); + LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet (populate) ", ze_rc, { return -1; } ); + + int deviceId = 0; + for(int did = 0; did < (int)deviceCount; did++) { + ze_device_properties_t device_properties; + zeDeviceGetProperties(allDevices[did], &device_properties); + if( ZE_DEVICE_TYPE_GPU != device_properties.type) { continue; } + gpuDevices[deviceId++] = allDevices[did]; + if( deviceId > maxDevices ) { + break; + } + } + free(allDevices); + allDevices = NULL; + + if( deviceId == 0) { + free(gpuDevices); + return 0; + } + deviceCount = deviceId; + + driver->devices = (device_t*)malloc(deviceCount * sizeof(device_t)); + + // Create context + ze_context_desc_t ctxtDesc = { + ZE_STRUCTURE_TYPE_CONTEXT_DESC, + NULL, + 0 + }; + ze_rc = zeContextCreate(driver->driver, &ctxtDesc, &driver->context); + LEVEL_ZERO_CHECK_ERROR( "zeContextCreate ", ze_rc, { continue; } ); + + int dpos = 0; + for(int did = 0; did < (int)deviceCount; did++) { + driver->devices[dpos].driver = driver; + if( init_device(&driver->devices[dpos], gpuDevices[did]) < 0 ) { + continue; + } + dpos++; + } + deviceCount = dpos; + + driver->nb_devices = deviceCount; + if(deviceCount == 0) { + free(driver->devices); + driver->devices = NULL; + return 0; + } + free(gpuDevices); + + driver->swp = sycl_wrapper_platform_create(driver->driver); + sycl_wrapper_device_t *swd[deviceCount]; + for(int did = 0; did < (int)deviceCount; did++) { + driver->devices[did].swd = sycl_wrapper_device_create(driver->devices[did].device); + swd[did] = driver->devices[did].swd; + } + sycl_wrapper_platform_add_context(driver->swp, driver->context, swd, deviceCount); + for(int did = 0; did < (int)deviceCount; did++) { + for(int sid = 0; sid < NB_STREAMS; sid++) { + driver->devices[did].streams[sid].swq = sycl_wrapper_queue_create(driver->swp, driver->devices[did].swd, driver->devices[did].streams[sid].cq); + } + } + + return deviceCount; +} + +static void *allocate_workspace(device_t *device, size_t size) +{ + ze_result_t status; + ze_device_properties_t devProperties; + ze_device_memory_properties_t *devMemProperties; + ze_device_memory_access_properties_t memAccessProperties; + void *device_ptr; + uint32_t count = 0; + int memIndex = -1; + + status = zeDeviceGetMemoryAccessProperties(device->device, &memAccessProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryAccessProperties ", status, { return NULL; }); + if( 0 == (ZE_MEMORY_ACCESS_CAP_FLAG_RW & memAccessProperties.deviceAllocCapabilities) ) { + fprintf(stderr, "Device does not have memory allocation capabilities with RW access\n"); + return NULL; + } + status = zeDeviceGetProperties(device->device, &devProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetProperties ", status, { return NULL; }); + status = zeDeviceGetMemoryProperties(device->device, &count, NULL); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties (count) ", status, { return NULL; }); + devMemProperties = (ze_device_memory_properties_t*)malloc(count * sizeof(ze_device_memory_properties_t)); + status = zeDeviceGetMemoryProperties(device->device, &count, devMemProperties); + LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties (populate) ", status, { free(devMemProperties); return NULL; }); + for(int i = 0; i < (int)count; i++) { + // TODO: better approach would be to keep a list of pointers? + // for now we just take the memory that has the highest amount of memory available + if( memIndex == -1 || devMemProperties[memIndex].totalSize < devMemProperties[i].totalSize) + memIndex = i; + } + free(devMemProperties); devMemProperties = NULL; + + if( size > devMemProperties[memIndex].totalSize ) { + /** Handle the case of jokers who require more than 100% of memory, + * and eleventh case of computer scientists who don't know how + * to divide a number by another + */ + fprintf(stderr, "Requested %zd bytes on LEVEL_ZERO device, but only %zd bytes are available -- Returning NULL\n", + size, devMemProperties[memIndex].totalSize); + return NULL; + } + ze_device_mem_alloc_desc_t memAllocDesc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = ZE_DEVICE_MEM_ALLOC_FLAG_BIAS_UNCACHED, + .ordinal = memIndex + }; + + status = zeMemAllocDevice(device->driver->context, &memAllocDesc, size, 128, + device->device, &device_ptr); + LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); + return device_ptr; +} + +int main(int argc, char *argv[]) +{ + ze_result_t ze_rc; + uint32_t driverCount = 0; + driver_t *drivers; + ze_driver_handle_t *allDrivers; + int max_devices = 1024*1024, nb_devices = 0; + void **device_workspaceA, **device_workspaceC; + + if(argc > 1) { + max_devices = atoi(argv[1]); + if(max_devices <= 0) { + fprintf(stderr, "USAGE: %s []\n", argv[0]); + return EXIT_FAILURE; + } + } + + // Discover all the driver instances + ze_rc = zeDriverGet(&driverCount, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeDriverGet (count) ", ze_rc, { return 1; } ); + fprintf(stderr, "STATUS: found %d drivers\n", driverCount); + drivers = malloc(driverCount * sizeof(driver_t)); + allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); + ze_rc = zeDriverGet(&driverCount, allDrivers); + LEVEL_ZERO_CHECK_ERROR( "zeDriverGet (populate) ", ze_rc, { return 1; } ); + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + int nb; + drivers[driverId].driver = allDrivers[driverId]; + if( (nb = init_driver(&drivers[driverId], max_devices)) <= 0 ) { + fprintf(stderr, "%d device found in driver %d... Bailing out\n", nb_devices, driverId); + return 1; + } else { + fprintf(stderr, "STATUS: there are %d GPU devices in driver %d\n", nb, driverId); + nb_devices += nb; + } + } + free(allDrivers); + fprintf(stderr, "%d devices found and initialized\n", nb_devices); + + //Allocate GPU memory for each device + device_workspaceA = (void**)malloc(sizeof(void*)*nb_devices); + device_workspaceC = (void**)malloc(sizeof(void*)*nb_devices); + int did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_workspaceA[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N); + device_workspaceC[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N); + did++; + } + } + + //Do a GEMM (blocking) on each device, and wait for its completion -- yes, memory is not initialized. + did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspace[did]) { + fprintf(stderr, "STATUS: Ready to submit GEMM on device %d of driver %d\n", deviceId, driverId); + dpcpp_kernel_GEMM(device->streams[2].sq, &((double*)device_workspace[did])[0], &((double*)device_workspace[did])[N*N*sizeof(double)], N); + fprintf(stderr, "STATUS: GEMM submitted on device %d of driver %d\n", deviceId, driverId); + + ze_rc = zeCommandListAppendSignalEvent( device->streams[2].cl, device->streams[2].events[0] ); + assert(ZE_RESULT_SUCCESS == ze_rc); + ze_rc = zeCommandListClose(device->streams[2].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[2].cq, 1, &device->streams[2].cl, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + ze_rc = zeCommandListReset(device->streams[2].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + + do { + ze_rc = zeEventQueryStatus(device->streams[2].events[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: GEMM ended on device %d of driver %d\n", deviceId, driverId); + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeEventQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(1000); + } + } while(1); + } else { + fprintf(stderr, "Skipping device %d which failed at allocating data\n", did); + } + did++; + } + } + + return EXIT_SUCCESS; +} From e33cc3af46c6baea1d31478aa149ada9d6fda193 Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Tue, 25 Oct 2022 15:50:01 -0400 Subject: [PATCH 08/15] Rebase the entire Level Zero driver based on the susbsystem test Buffer interface is not required. We can use the USM OneMKL interface, it seems to work ok. Need to check for performance. We cannot mix immediate and non-immediate command lists apparently. Or at least it makes the passing of command queues unreliable There is an exception in data.c how we handle GPU copies, it must be ported to Level Zero too. The Level Zero runtime has a atexit procedure to delete command queues, and this seems to conflict with our own actions to delete the command queues... --- parsec/data.c | 2 +- parsec/interfaces/ptg/ptg-compiler/jdf2c.c | 6 +- .../mca/device/level_zero/device_level_zero.h | 21 +++- .../level_zero/device_level_zero_component.c | 94 ++++++++++++++-- .../level_zero/device_level_zero_dpcpp.h | 35 ++++-- .../device_level_zero_dpcpp_interface.cpp | 65 ++++++++--- .../level_zero/device_level_zero_internal.h | 2 +- .../level_zero/device_level_zero_module.c | 102 ++++++------------ tests/dsl/ptg/cuda/CMakeLists.txt | 12 +-- tests/dsl/ptg/cuda/nvlink.jdf | 4 +- tests/dsl/ptg/cuda/nvlink_wrapper.c | 13 ++- tests/subsystem/level_zero/kernel.dpcpp.C | 10 +- tests/subsystem/level_zero/main.c | 11 +- 13 files changed, 244 insertions(+), 133 deletions(-) diff --git a/parsec/data.c b/parsec/data.c index 0f022c008..c83230bd3 100644 --- a/parsec/data.c +++ b/parsec/data.c @@ -94,7 +94,7 @@ static void parsec_data_destruct(parsec_data_t* obj ) #endif /* defined(PARSEC_DEBUG_PARANOID) */ assert(obj->super.obj_reference_count > 1); parsec_data_copy_detach( obj, copy, i ); - if ( !(device->type & PARSEC_DEV_CUDA) ){ + if ( !(device->type & PARSEC_DEV_CUDA) && !(device->type & PARSEC_DEV_LEVEL_ZERO) ){ /** * GPU copies are normally stored in LRU lists, and must be * destroyed by the release list to free the memory on the device diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index e50b8924d..dc5bc25a0 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -7037,10 +7037,12 @@ static void jdf_generate_code_hook_dpcpp(const jdf_t *jdf, " parsec_device_level_zero_module_t *level_zero_device = reinterpret_cast(gpu_device);\n" " parsec_level_zero_exec_stream_t *level_zero_stream = reinterpret_cast(gpu_stream);\n" " __parsec_%s_internal_taskpool_t *__parsec_tp = reinterpret_cast<__parsec_%s_internal_taskpool_t *>(this_task->taskpool);\n" - " parsec_dpcpp_object_t *parsec_dpcpp = reinterpret_cast(level_zero_stream->dpcpp_obj);\n" + " parsec_sycl_wrapper_platform_t *parsec_sycl_platform = level_zero_device->driver->swp;\n" + " parsec_sycl_wrapper_device_t *parsec_sycl_device = level_zero_device->swd;\n" + " parsec_sycl_wrapper_queue_t *parsec_sycl_queue = level_zero_stream->swq;\n" "\n" "%s\n" - " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)level_zero_device; (void)level_zero_stream;\n", + " (void)gpu_device; (void)gpu_stream; (void)__parsec_tp; (void)level_zero_device; (void)level_zero_stream; (void)parsec_sycl_platform; (void)parsec_sycl_device; (void)parsec_sycl_queue;\n", jdf_basename, f->fname, jdf_basename, f->fname, parsec_get_name(jdf, f, "task_t"), parsec_get_name(jdf, f, "task_t"), diff --git a/parsec/mca/device/level_zero/device_level_zero.h b/parsec/mca/device/level_zero/device_level_zero.h index 19c13715b..e7f06ab61 100644 --- a/parsec/mca/device/level_zero/device_level_zero.h +++ b/parsec/mca/device/level_zero/device_level_zero.h @@ -11,6 +11,7 @@ #if defined(PARSEC_HAVE_LEVEL_ZERO) #include "parsec/mca/device/device.h" #include "parsec/mca/device/device_gpu.h" +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" #include @@ -25,6 +26,9 @@ typedef struct parsec_level_zero_exec_stream_s parsec_level_zero_exec_stream_t; struct parsec_device_level_zero_module_s; typedef struct parsec_device_level_zero_module_s parsec_device_level_zero_module_t; +struct parsec_device_level_zero_driver_s; +typedef struct parsec_device_level_zero_driver_s parsec_device_level_zero_driver_t; + struct parsec_level_zero_workspace_s; typedef struct parsec_level_zero_workspace_s parsec_level_zero_workspace_t; @@ -34,12 +38,19 @@ struct parsec_level_zero_task_s { parsec_gpu_task_t super; }; -struct parsec_device_level_zero_module_s { - parsec_device_gpu_module_t super; - uint8_t level_zero_index; +struct parsec_device_level_zero_driver_s { ze_driver_handle_t ze_driver; - ze_device_handle_t ze_device; ze_context_handle_t ze_context; + uint32_t ref_count; + parsec_sycl_wrapper_platform_t *swp; +}; + +struct parsec_device_level_zero_module_s { + parsec_device_gpu_module_t super; + uint8_t level_zero_index; + parsec_device_level_zero_driver_t *driver; + ze_device_handle_t ze_device; + parsec_sycl_wrapper_device_t *swd; }; PARSEC_OBJ_CLASS_DECLARATION(parsec_device_level_zero_module_t); @@ -55,7 +66,7 @@ struct parsec_level_zero_exec_stream_s { ze_event_pool_handle_t ze_event_pool; ze_command_list_handle_t level_zero_cl; ze_command_queue_handle_t level_zero_cq; - void *dpcpp_obj; + parsec_sycl_wrapper_queue_t *swq; }; diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c index db78be3d6..0f1cbc231 100644 --- a/parsec/mca/device/level_zero/device_level_zero_component.c +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -86,6 +86,7 @@ mca_base_component_t * device_level_zero_static_component(void) static int device_level_zero_component_query(mca_base_module_t **module, int *priority) { int i, j, rc; + ze_result_t ze_rc; ze_device_handle_t *devices = NULL; ze_driver_handle_t *allDrivers = NULL; @@ -101,10 +102,12 @@ static int device_level_zero_component_query(mca_base_module_t **module, int *pr uint32_t totalDeviceCount = 0, maxDeviceCount = 0; // Discover all the driver instances - zeDriverGet(&driverCount, NULL); + ze_rc = zeDriverGet(&driverCount, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, { return MCA_ERR_NOT_AVAILABLE; } ); allDrivers = malloc(driverCount * sizeof(ze_driver_handle_t)); - zeDriverGet(&driverCount, allDrivers); + ze_rc = zeDriverGet(&driverCount, allDrivers); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDriverGet ", ze_rc, { free(allDrivers); return MCA_ERR_NOT_AVAILABLE; } ); for(uint32_t did = 0; did < driverCount; ++did ) { uint32_t deviceCount = 0; @@ -125,27 +128,56 @@ static int device_level_zero_component_query(mca_base_module_t **module, int *pr i = j = 0; for(uint32_t did = 0; i < parsec_device_level_zero_enabled && did < driverCount; ++did ) { uint32_t deviceCount = maxDeviceCount; - zeDeviceGet(allDrivers[did], &deviceCount, devices); + parsec_device_level_zero_driver_t *driver = malloc(sizeof(parsec_device_level_zero_driver_t)); + + driver->ze_driver = allDrivers[did]; + driver->ref_count = 0; + driver->swp = NULL; + + // Create context + ze_context_desc_t ctxtDesc = { + ZE_STRUCTURE_TYPE_CONTEXT_DESC, + NULL, + 0 + }; + ze_result_t ze_rc = zeContextCreate(driver->ze_driver, &ctxtDesc, &driver->ze_context); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeContextCreate ", ze_rc, { free(driver); continue; } ); + + ze_rc = zeDeviceGet(allDrivers[did], &deviceCount, devices); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet ", ze_rc, { zeContextDestroy(driver->ze_context); free(driver); continue; } ); + for(uint32_t devid = 0; i < parsec_device_level_zero_enabled && devid < deviceCount; devid++) { ze_device_properties_t device_properties; - zeDeviceGetProperties(devices[devid], &device_properties); + ze_rc = zeDeviceGetProperties(devices[devid], &device_properties); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeDeviceGetProperties ", ze_rc, { continue; } ); if( ZE_DEVICE_TYPE_GPU != device_properties.type) { continue; } if( !((1 << i) & level_zero_mask) ) { i++; continue; } - rc = parsec_level_zero_module_init(i, allDrivers[did], devices[devid], &device_properties, + rc = parsec_level_zero_module_init(i, driver, devices[devid], &device_properties, &parsec_device_level_zero_component.modules[j]); if( PARSEC_SUCCESS != rc ) { assert( NULL == parsec_device_level_zero_component.modules[j] ); continue; } + driver->ref_count++; parsec_device_level_zero_component.modules[j]->component = &parsec_device_level_zero_component; j++; /* next available spot */ parsec_device_level_zero_component.modules[j] = NULL; i++; } + + if( driver->ref_count == 0 ) { + zeContextDestroy(driver->ze_context); + free(driver); + } } - } else - parsec_device_level_zero_component.modules = NULL; - } else + parsec_device_level_zero_enabled = j; + if(0 == parsec_device_level_zero_enabled) { + free( parsec_device_level_zero_component.modules ); + } + } + } + + if(0 == parsec_device_level_zero_enabled ) parsec_device_level_zero_component.modules = NULL; if(NULL != devices) @@ -153,6 +185,47 @@ static int device_level_zero_component_query(mca_base_module_t **module, int *pr if(NULL != allDrivers) free(allDrivers); + /* SYCL wrappers must be built once all devices and queues and contexts and drivers are up, + * and they must be built in the following order: driver wrapper, then device wrappers for this + * driver, then context wrapper for this driver, then queue wrappers for each queue. + * Also, it MUST be one to one: a single device wrapper per device, a single context wrapper + * per context etc... */ + for(int did = 0; did < parsec_device_level_zero_enabled; did++) { + if(NULL == parsec_device_level_zero_component.modules[did]) + continue; + parsec_device_level_zero_module_t *module = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did]; + if(NULL != module->driver->swp) + continue; + + module->driver->swp = parsec_sycl_wrapper_platform_create(module->driver->ze_driver); + int nbdev = 1; + for(int i = 1; did + i < parsec_device_level_zero_enabled; i++) { + if( ((parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i])->driver == module->driver ) + nbdev++; + } + parsec_sycl_wrapper_device_t **devices = (parsec_sycl_wrapper_device_t **)malloc(nbdev*sizeof(parsec_sycl_wrapper_device_t*)); + for(int i = 0, j = 0; j < nbdev; i++) { + parsec_device_level_zero_module_t *mod2 = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i]; + if( mod2->driver == module->driver ) { + mod2->swd = parsec_sycl_wrapper_device_create(mod2->ze_device); + devices[j++] = mod2->swd; + } + } + parsec_sycl_wrapper_platform_add_context(module->driver->swp, module->driver->ze_context, devices, nbdev); + free(devices); + for(int i = 0, j = 0; j < nbdev; i++) { + parsec_device_level_zero_module_t *mod2 = (parsec_device_level_zero_module_t *)parsec_device_level_zero_component.modules[did+i]; + if( mod2->driver == module->driver ) { + j++; + for(int s = 0; s < mod2->super.max_exec_streams; s++) { + parsec_level_zero_exec_stream_t *exec_stream = (parsec_level_zero_exec_stream_t *)mod2->super.exec_stream[s]; + if(NULL != exec_stream->level_zero_cq) + exec_stream->swq = parsec_sycl_wrapper_queue_create(module->driver->swp, mod2->swd, exec_stream->level_zero_cq); + } + } + } + } + parsec_gpu_enable_debug(); /* module type should be: const mca_base_module_t ** */ @@ -310,6 +383,11 @@ static int device_level_zero_component_close(void) "GPU[%d] Failed to release resources on LEVEL_ZERO device\n", cdev->level_zero_index); } + if(0 == --cdev->driver->ref_count) { + ze_result_t ze_rc = zeContextDestroy(cdev->driver->ze_context); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeContextDestroy ", ze_rc, {}); + parsec_sycl_wrapper_platform_destroy(cdev->driver->swp); + } /* unregister the device from PaRSEC */ rc = parsec_mca_device_remove((parsec_device_module_t*)cdev); diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp.h b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h index 00c00cc63..ed3a77998 100644 --- a/parsec/mca/device/level_zero/device_level_zero_dpcpp.h +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp.h @@ -1,24 +1,39 @@ #ifndef PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H #define PARSEC_DEVICE_LEVEL_ZERO_DPCPP_H +typedef struct parsec_sycl_wrapper_platform_s parsec_sycl_wrapper_platform_t; +typedef struct parsec_sycl_wrapper_device_s parsec_sycl_wrapper_device_t; +typedef struct parsec_sycl_wrapper_queue_s parsec_sycl_wrapper_queue_t; + +#include + #if defined(c_plusplus) || defined(__cplusplus) -#include "sycl/ext/oneapi/backend/level_zero.hpp" +#include -typedef struct { +struct parsec_sycl_wrapper_platform_s { sycl::platform platform; - sycl::device device; sycl::context context; - sycl::queue queue; -} parsec_dpcpp_object_t; +}; + +struct parsec_sycl_wrapper_device_s { + sycl::device device; +}; + +struct parsec_sycl_wrapper_queue_s { + sycl::queue queue; +}; extern "C" { #endif -void * parsec_dpcpp_queue_create(ze_driver_handle_t ze_driver, - ze_device_handle_t ze_device, - ze_context_handle_t ze_context, - ze_command_queue_handle_t ze_queue); -int parsec_dpcpp_queue_destroy(void *_dpcpp_obj); +parsec_sycl_wrapper_platform_t *parsec_sycl_wrapper_platform_create(ze_driver_handle_t ze_driver); +void parsec_sycl_wrapper_platform_add_context(parsec_sycl_wrapper_platform_t *swp, ze_context_handle_t ze_context, parsec_sycl_wrapper_device_t **swd, uint32_t num_device); +parsec_sycl_wrapper_device_t *parsec_sycl_wrapper_device_create(ze_device_handle_t ze_device); +parsec_sycl_wrapper_queue_t *parsec_sycl_wrapper_queue_create(parsec_sycl_wrapper_platform_t *swp, parsec_sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue); + +int parsec_sycl_wrapper_platform_destroy(parsec_sycl_wrapper_platform_t *swp); +int parsec_sycl_wrapper_device_destroy(parsec_sycl_wrapper_device_t *swd); +int parsec_sycl_wrapper_queue_destroy(parsec_sycl_wrapper_queue_t *swq); #if defined(c_plusplus) || defined(__cplusplus) } diff --git a/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp b/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp index 0868e9911..bafd03cca 100644 --- a/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp +++ b/parsec/mca/device/level_zero/device_level_zero_dpcpp_interface.cpp @@ -1,28 +1,59 @@ -#include "level_zero/ze_api.h" -#include "device_level_zero_dpcpp.h" +#include +#include "parsec/mca/device/level_zero/device_level_zero_dpcpp.h" +#include "parsec/mca/device/level_zero/device_level_zero_internal.h" -void * parsec_dpcpp_queue_create(ze_driver_handle_t ze_driver, - ze_device_handle_t ze_device, - ze_context_handle_t ze_context, - ze_command_queue_handle_t ze_queue) +parsec_sycl_wrapper_platform_t *parsec_sycl_wrapper_platform_create(ze_driver_handle_t ze_driver) +{ + parsec_sycl_wrapper_platform_t *res = new parsec_sycl_wrapper_platform_t; + res->platform = sycl::make_platform(ze_driver); + return res; +} + +void parsec_sycl_wrapper_platform_add_context(parsec_sycl_wrapper_platform_t *swp, ze_context_handle_t ze_context, parsec_sycl_wrapper_device_t **swd, uint32_t num_device) { - parsec_dpcpp_object_t *dpcpp_obj; std::vectordevices; - dpcpp_obj = new parsec_dpcpp_object_t; - dpcpp_obj->platform = sycl::level_zero::make(ze_driver); - dpcpp_obj->device = sycl::level_zero::make(dpcpp_obj->platform, ze_device); - devices.push_back(dpcpp_obj->device); - dpcpp_obj->context = sycl::level_zero::make(devices, ze_context); - dpcpp_obj->queue = sycl::level_zero::make(dpcpp_obj->context, ze_queue); + for(uint32_t i = 0; i < num_device; i++) { + devices.push_back(swd[i]->device); + } + sycl::backend_input_t hContextInteropInput = {ze_context, devices}; + swp->context = sycl::make_context(hContextInteropInput); +} - return static_cast(dpcpp_obj); +parsec_sycl_wrapper_device_t *parsec_sycl_wrapper_device_create(ze_device_handle_t ze_device) +{ + parsec_sycl_wrapper_device_t *res = new parsec_sycl_wrapper_device_t; + + res->device = sycl::make_device(ze_device); + + return res; } -int parsec_dpcpp_queue_destroy(void *_dpcpp_obj) +parsec_sycl_wrapper_queue_t *parsec_sycl_wrapper_queue_create(parsec_sycl_wrapper_platform_t *swp, parsec_sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue) { - parsec_dpcpp_object_t *dpcpp_obj = reinterpret_cast(_dpcpp_obj); - delete dpcpp_obj; + parsec_sycl_wrapper_queue_t *swq = new parsec_sycl_wrapper_queue_t; + sycl::backend_input_t hQueueInteropInput = { ze_queue, swd->device }; + swq->queue = sycl::make_queue(hQueueInteropInput, swp->context); + return swq; +} +int parsec_sycl_wrapper_platform_destroy(parsec_sycl_wrapper_platform_t *swp) +{ + if(nullptr != swp) + delete swp; + return 0; +} + +int parsec_sycl_wrapper_device_destroy(parsec_sycl_wrapper_device_t *swd) +{ + if(nullptr != swd) + delete swd; + return 0; +} + +int parsec_sycl_wrapper_queue_destroy(parsec_sycl_wrapper_queue_t *swq) +{ + if(nullptr != swq) + delete swq; return 0; } diff --git a/parsec/mca/device/level_zero/device_level_zero_internal.h b/parsec/mca/device/level_zero/device_level_zero_internal.h index e6ff2cf16..b2bd8de42 100644 --- a/parsec/mca/device/level_zero/device_level_zero_internal.h +++ b/parsec/mca/device/level_zero/device_level_zero_internal.h @@ -58,7 +58,7 @@ PARSEC_DECLSPEC extern const parsec_device_module_t parsec_device_level_zero_mod ** GPU-DATA Specific Starts Here ** ****************************************************/ -int parsec_level_zero_module_init( int device_id, ze_driver_handle_t ze_driver, ze_device_handle_t ze_device, +int parsec_level_zero_module_init( int device_id, parsec_device_level_zero_driver_t *driver, ze_device_handle_t ze_device, ze_device_properties_t *device_properties, parsec_device_module_t** module ); int parsec_level_zero_module_fini(parsec_device_module_t* device); diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c index 0bdf9869e..3f014fe3a 100644 --- a/parsec/mca/device/level_zero/device_level_zero_module.c +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -150,7 +150,7 @@ parsec_device_level_zero_detach( parsec_device_module_t* device, return parsec_mca_device_remove(device); } -int parsec_level_zero_module_init( int dev_id, ze_driver_handle_t ze_driver, ze_device_handle_t ze_device, +int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t *driver, ze_device_handle_t ze_device, ze_device_properties_t *prop, parsec_device_module_t** module ) { int streaming_multiprocessor, len; @@ -178,23 +178,9 @@ int parsec_level_zero_module_init( int dev_id, ze_driver_handle_t ze_driver, ze_ device = &gpu_device->super; PARSEC_OBJ_CONSTRUCT(level_zero_device, parsec_device_level_zero_module_t); level_zero_device->level_zero_index = (uint8_t)dev_id; - level_zero_device->ze_driver = ze_driver; + level_zero_device->driver = driver; level_zero_device->ze_device = ze_device; - // Create context - ze_context_desc_t ctxtDesc = { - ZE_STRUCTURE_TYPE_CONTEXT_DESC, - NULL, - 0 - }; - - ze_rc = zeContextCreate(ze_driver, &ctxtDesc, &level_zero_device->ze_context); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeContextCreate ", ze_rc, - { - free(level_zero_device); - return PARSEC_ERROR; - } ); - len = asprintf(&gpu_device->super.name, "%s ZE(%d)", szName, dev_id); if(-1 == len) gpu_device->super.name = ""; @@ -255,35 +241,21 @@ int parsec_level_zero_module_init( int dev_id, ze_driver_handle_t ze_driver, ze_ ZE_COMMAND_QUEUE_MODE_DEFAULT, ZE_COMMAND_QUEUE_PRIORITY_NORMAL }; - if( j < 2 ) { - commandQueueDesc.ordinal = copyQueueGroupOrdinal; - ze_rc = zeCommandListCreateImmediate(level_zero_device->ze_context, level_zero_device->ze_device, - &commandQueueDesc, - &level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreateImmediate ", ze_rc, - {goto release_device;} ); - } else { - commandQueueDesc.ordinal = computeQueueGroupOrdinal; - ze_rc = zeCommandQueueCreate(level_zero_device->ze_context, level_zero_device->ze_device, - &commandQueueDesc, &level_zero_stream->level_zero_cq); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, - {goto release_device;} ); - ze_command_list_desc_t commandListDesc = { + /* CommandListImmediate would seem better for I/O, but mixing CLImmediate and CL+CQ + * seems to create some synchronization issues. */ + commandQueueDesc.ordinal = computeQueueGroupOrdinal; + ze_rc = zeCommandQueueCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, + &commandQueueDesc, &level_zero_stream->level_zero_cq); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, {goto release_device;} ); + ze_command_list_desc_t commandListDesc = { ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, NULL, computeQueueGroupOrdinal, 0 // flags - }; - ze_rc = zeCommandListCreate(level_zero_device->ze_context, level_zero_device->ze_device, - &commandListDesc, &level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, - {goto release_device;} ); - level_zero_stream->dpcpp_obj = parsec_dpcpp_queue_create(level_zero_device->ze_driver, - level_zero_device->ze_device, - level_zero_device->ze_context, - level_zero_stream->level_zero_cq); - assert(NULL != level_zero_stream->dpcpp_obj); - } + }; + ze_rc = zeCommandListCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, + &commandListDesc, &level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, {goto release_device;} ); exec_stream->workspace = NULL; PARSEC_OBJ_CONSTRUCT(&exec_stream->infos, parsec_info_object_array_t); parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream); @@ -305,7 +277,7 @@ int parsec_level_zero_module_init( int dev_id, ze_driver_handle_t ze_driver, ze_ ZE_EVENT_POOL_FLAG_HOST_VISIBLE, // all events in pool are visible to Host 1 // count }; - zeEventPoolCreate(level_zero_device->ze_context, &eventPoolDesc, 0, NULL, + zeEventPoolCreate(level_zero_device->driver->ze_context, &eventPoolDesc, 0, NULL, &level_zero_stream->ze_event_pool); /* and the corresponding events */ @@ -487,8 +459,7 @@ parsec_level_zero_module_fini(parsec_device_module_t* device) for( k = 0; k < exec_stream->max_events; k++ ) { assert( NULL == exec_stream->tasks[k] ); status = zeEventDestroy(level_zero_stream->events[k]); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "(parsec_level_zero_device_fini) level_zeroEventDestroy ", status, - {continue;} ); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(parsec_level_zero_device_fini) level_zeroEventDestroy ", status, {} ); } zeEventPoolDestroy(level_zero_stream->ze_event_pool); exec_stream->max_events = 0; @@ -496,17 +467,14 @@ parsec_level_zero_module_fini(parsec_device_module_t* device) free(exec_stream->tasks); exec_stream->tasks = NULL; free(exec_stream->fifo_pending); exec_stream->fifo_pending = NULL; /* Release the stream */ - if(k < 2) { - ze_result_t ze_rc; - ze_rc = zeCommandListDestroy(level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListDestroy ", ze_rc, - {continue;} ); - } else { - int rc; - rc = parsec_dpcpp_queue_destroy(level_zero_stream->dpcpp_obj); - assert(PARSEC_SUCCESS == rc); - level_zero_stream->dpcpp_obj = NULL; - } + ze_result_t ze_rc; + ze_rc = zeCommandListDestroy(level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListDestroy ", ze_rc, {} ); + /* Deleting the sycl queue wrapper and/or the command queue conflicts with the + * cleaning procedure of the Level Zero runtime... Didn't find a way to do it + * cleanly. Don't cleanup for now... */ + //parsec_sycl_wrapper_queue_destroy(level_zero_stream->swq); + level_zero_stream->swq = NULL; free(exec_stream->name); /* Release Info object array */ @@ -517,6 +485,8 @@ parsec_level_zero_module_fini(parsec_device_module_t* device) free(gpu_device->exec_stream); gpu_device->exec_stream = NULL; level_zero_device->level_zero_index = -1; + parsec_sycl_wrapper_device_destroy(level_zero_device->swd); + level_zero_device->swd = NULL; /* Cleanup the GPU memory. */ PARSEC_OBJ_DESTRUCT(&gpu_device->gpu_mem_lru); @@ -676,7 +646,7 @@ parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* level_zero_ total_size = (size_t)((int)(.9*initial_free_mem / eltsize)) * eltsize; mem_elem_per_gpu = total_size / eltsize; } - status = zeMemAllocDevice(level_zero_device->ze_context, &memAllocDesc, total_size, alignment, + status = zeMemAllocDevice(level_zero_device->driver->ze_context, &memAllocDesc, total_size, alignment, level_zero_device->ze_device, &base_ptr); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, ({ parsec_warning("GPU[%s] Allocating %zu bytes of memory on the GPU device failed", @@ -795,7 +765,7 @@ parsec_level_zero_memory_release( parsec_device_level_zero_module_t* level_zero_ #if !defined(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE) assert( NULL != level_zero_device->super.memory ); void* ptr = zone_malloc_fini(&level_zero_device->super.memory); - status = zeMemFree(level_zero_device->ze_context, ptr); + status = zeMemFree(level_zero_device->driver->ze_context, ptr); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, { parsec_warning("Failed to free the GPU backend memory."); } ); #endif @@ -2009,17 +1979,13 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, */ rc = zeCommandListAppendSignalEvent( level_zero_stream->level_zero_cl, level_zero_stream->events[stream->start] ); assert(ZE_RESULT_SUCCESS == rc); - if( stream->start >= 2 ) { - ze_result_t ze_rc = zeCommandListClose(level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, - { } ); - ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->level_zero_cl, NULL); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, - { } ); - ze_rc = zeCommandListReset(level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, - { } ); - } + ze_result_t ze_rc = zeCommandListClose(level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { } ); + ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->level_zero_cl, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { } ); + ze_rc = zeCommandListReset(level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { } ); + stream->tasks[stream->start] = task; stream->start = (stream->start + 1) % stream->max_events; PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, diff --git a/tests/dsl/ptg/cuda/CMakeLists.txt b/tests/dsl/ptg/cuda/CMakeLists.txt index 7904e5671..69a493a9c 100644 --- a/tests/dsl/ptg/cuda/CMakeLists.txt +++ b/tests/dsl/ptg/cuda/CMakeLists.txt @@ -1,20 +1,20 @@ if(PARSEC_HAVE_LEVEL_ZERO) if(NOT TARGET MKL::MKL_DPCPP) find_package(MKL) -if(NOT MKL_FOUND) - message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") -endif(NOT MKL_FOUND) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) endif(NOT TARGET MKL::MKL_DPCPP) if(TARGET MKL::MKL_DPCPP) -message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") include(ParsecCompilePTG) parsec_addtest_executable(C nvlink SOURCES nvlink_main.c nvlink_wrapper.c) target_include_directories(nvlink PRIVATE $<$:${CMAKE_CURRENT_SOURCE_DIR}>) -target_ptg_source_ex(SOURCE "nvlink.jdf" MODE PRIVATE TARGET nvlink DESTINATION_DPCPP "nvlink.dpcpp.C") + target_ptg_source_ex(SOURCE "nvlink.jdf" MODE PRIVATE TARGET nvlink DESTINATION_DPCPP "nvlink.dpcpp.C") target_link_libraries(nvlink PRIVATE MKL::MKL_DPCPP level_zero::ze_loader) else(TARGET MKL::MKL_DPCPP) -message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink_lz test with DPCPP is disabled") + message(STATUS "Target MKL::MKL_DPCPP is not defined, nvlink_lz test with DPCPP is disabled") endif(TARGET MKL::MKL_DPCPP) elseif(PARSEC_HAVE_CUDA) if( NOT TARGET CUDA::cublas ) diff --git a/tests/dsl/ptg/cuda/nvlink.jdf b/tests/dsl/ptg/cuda/nvlink.jdf index 8ce2f6ebb..f67b8a22a 100644 --- a/tests/dsl/ptg/cuda/nvlink.jdf +++ b/tests/dsl/ptg/cuda/nvlink.jdf @@ -163,7 +163,7 @@ BODY [type=DPCPP double alpha=0.0; double beta=1.0; try { - oneapi::mkl::blas::gemm(parsec_dpcpp->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + oneapi::mkl::blas::gemm(parsec_sycl_queue->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, descA->super.mb, descA->super.nb, descA->super.mb, alpha, static_cast(A), descA->super.mb, static_cast(A), descA->super.mb, @@ -238,7 +238,7 @@ BODY [type=DPCPP double alpha=0.0; double beta=1.0; try { - oneapi::mkl::blas::gemm(parsec_dpcpp->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + oneapi::mkl::blas::gemm(parsec_sycl_queue->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, descA->super.mb, descA->super.nb, descA->super.mb, alpha, static_cast(A), descA->super.mb, static_cast(A), descA->super.mb, diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index 269df7ba9..40835aeed 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -90,7 +90,7 @@ __parsec_nvlink_destructor( parsec_nvlink_taskpool_t* nvlink_taskpool) PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaFree ", status, {} ); #elif defined(PARSEC_HAVE_LEVEL_ZERO) parsec_device_level_zero_module_t *level_zero_device = (parsec_device_level_zero_module_t *)device; - ze_result_t status = zeMemFree(level_zero_device->ze_context, gpu_copy->device_private); + ze_result_t status = zeMemFree(level_zero_device->driver->ze_context, gpu_copy->device_private); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemFree ", status, {} ); #else free(gpu_copy->device_private); @@ -220,7 +220,7 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb ze_result_t status = zeDeviceGetMemoryProperties(level_zero_device->ze_device, &count, &devMemProperties); PARSEC_LEVEL_ZERO_CHECK_ERROR("zeDeviceGetMemoryProperties ", status, { return NULL; }); assert(count >= 1); - assert(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) + 128 <= devMemProperties.totalSize); + assert(mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE) + 128 <= (int)devMemProperties.totalSize); ze_device_mem_alloc_desc_t memAllocDesc = { .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, .pNext = NULL, @@ -228,7 +228,7 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb .ordinal = 0 }; /* Allocate memory on it, for one tile */ - status = zeMemAllocDevice(level_zero_device->ze_context, &memAllocDesc, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, + status = zeMemAllocDevice(level_zero_device->driver->ze_context, &memAllocDesc, mb*mb*parsec_datadist_getsizeoftype(PARSEC_MATRIX_DOUBLE), 128, level_zero_device->ze_device, &gpu_copy->device_private); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeMemAllocDevice ", status, { return NULL; } ); #else @@ -251,6 +251,13 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; status = zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); + status = zeCommandListClose(level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", status, { return NULL; } ); + status = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->level_zero_cl, NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", status, { return NULL; } ); + status = zeCommandListReset(level_zero_stream->level_zero_cl); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", status, { return NULL; } ); + while(1) { status = zeEventQueryStatus(copySignalEvent); if(status == ZE_RESULT_SUCCESS) diff --git a/tests/subsystem/level_zero/kernel.dpcpp.C b/tests/subsystem/level_zero/kernel.dpcpp.C index 839f99627..eea5c23bb 100644 --- a/tests/subsystem/level_zero/kernel.dpcpp.C +++ b/tests/subsystem/level_zero/kernel.dpcpp.C @@ -20,18 +20,18 @@ int dpcpp_kernel_GEMM(void *_sw, double alpha=0.0; double beta=1.0; - sycl::backend_input_t> hBufferInteropInputA = { (void*)A, sycl::ext::oneapi::level_zero::ownership::keep }; +/* sycl::backend_input_t> hBufferInteropInputA = { (void*)A, sycl::ext::oneapi::level_zero::ownership::keep }; sycl::buffer bA = sycl::make_buffer(hBufferInteropInputA, swp->context); auto bbA = bA.reinterpret(sycl::range<1>(mb*mb)); sycl::backend_input_t> hBufferInteropInputC = { (void*)C, sycl::ext::oneapi::level_zero::ownership::keep }; sycl::buffer bC = sycl::make_buffer(hBufferInteropInputC, swp->context); - auto bbC = bC.reinterpret(sycl::range<1>(mb*mb)); + auto bbC = bC.reinterpret(sycl::range<1>(mb*mb));*/ try { oneapi::mkl::blas::gemm(swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, mb, mb, mb, - alpha, bbA, mb, - bbA, mb, - beta, bbC, mb); + alpha, A, mb, + A, mb, + beta, C, mb); } catch (const oneapi::mkl::invalid_argument &e) { fprintf(stderr, "OneAPI MKL BLAS GEMM throws invalid argument exception"); } catch (const oneapi::mkl::unsupported_device &e) { diff --git a/tests/subsystem/level_zero/main.c b/tests/subsystem/level_zero/main.c index 6adb371ef..e2444fae4 100644 --- a/tests/subsystem/level_zero/main.c +++ b/tests/subsystem/level_zero/main.c @@ -340,7 +340,7 @@ int main(int argc, char *argv[]) int did = 0; for(int driverId = 0; driverId < (int)driverCount; driverId++) { for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { - device_workspaceA[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N); + device_workspaceA[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N*2); device_workspaceC[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N); did++; } @@ -351,10 +351,11 @@ int main(int argc, char *argv[]) for(int driverId = 0; driverId < (int)driverCount; driverId++) { for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { device_t *device = &drivers[driverId].devices[deviceId]; - if(NULL != device_workspace[did]) { - fprintf(stderr, "STATUS: Ready to submit GEMM on device %d of driver %d\n", deviceId, driverId); - dpcpp_kernel_GEMM(device->streams[2].sq, &((double*)device_workspace[did])[0], &((double*)device_workspace[did])[N*N*sizeof(double)], N); - fprintf(stderr, "STATUS: GEMM submitted on device %d of driver %d\n", deviceId, driverId); + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Ready to submit GEMM[%d] on device %d of driver %d\n", run, deviceId, driverId); + fprintf(stderr, "STATUS: Context of driver %d is %s\n", driverId, zeContextGetStatus(drivers[driverId].context) == ZE_RESULT_SUCCESS ? "Fine" : "Broken"); + dpcpp_kernel_GEMM(device->driver->swp, device->swd, device->streams[2].swq, (double*)device_workspaceA[did], &(((double*)device_workspaceC[did])[N*N]), N); + fprintf(stderr, "STATUS: GEMM[%d] submitted on device %d of driver %d\n", run, deviceId, driverId); ze_rc = zeCommandListAppendSignalEvent( device->streams[2].cl, device->streams[2].events[0] ); assert(ZE_RESULT_SUCCESS == ze_rc); From ddf9b253bfaa27ed550a42a99aa50da4f5e0a73b Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Thu, 27 Oct 2022 21:58:13 +0000 Subject: [PATCH 09/15] Porting of the DTD GEMM test to Level Zero NULL is not a valid MPI datatype when compiling with a clone of MPICH. The value doesn't matter in this case, just cast --- parsec/remote_dep_mpi.c | 4 +- tests/dsl/dtd/CMakeLists.txt | 23 +- tests/dsl/dtd/dtd_test_simple_gemm_lz.c | 564 ++++++++++++++++++ .../dtd_test_simple_gemm_lz_kernel.dpcpp.C | 75 +++ 4 files changed, 661 insertions(+), 5 deletions(-) create mode 100644 tests/dsl/dtd/dtd_test_simple_gemm_lz.c create mode 100644 tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C diff --git a/parsec/remote_dep_mpi.c b/parsec/remote_dep_mpi.c index 8d8759cb1..2916e41ae 100644 --- a/parsec/remote_dep_mpi.c +++ b/parsec/remote_dep_mpi.c @@ -1717,7 +1717,7 @@ remote_dep_mpi_put_start(parsec_execution_stream_t* es, int dtt_size; parsec_type_size(dtt, &dtt_size); parsec_ce.mem_register(dataptr, PARSEC_MEM_TYPE_CONTIGUOUS, - -1, NULL, // TODO JS: this interface is so broken, fix it! + -1, (parsec_datatype_t)NULL, // TODO JS: this interface is so broken, fix it! dtt_size, // TODO JS: what about nbdtt? Is it ok to ignore it?! &source_memory_handle, &source_memory_handle_size); @@ -2061,7 +2061,7 @@ static void remote_dep_mpi_get_start(parsec_execution_stream_t* es, int dtt_size; parsec_type_size(dtt, &dtt_size); parsec_ce.mem_register(PARSEC_DATA_COPY_GET_PTR(deps->output[k].data.data), PARSEC_MEM_TYPE_CONTIGUOUS, - -1, NULL, + -1, (parsec_datatype_t)NULL, dtt_size, &receiver_memory_handle, &receiver_memory_handle_size); diff --git a/tests/dsl/dtd/CMakeLists.txt b/tests/dsl/dtd/CMakeLists.txt index 4246df4ac..d642c342a 100644 --- a/tests/dsl/dtd/CMakeLists.txt +++ b/tests/dsl/dtd/CMakeLists.txt @@ -24,7 +24,25 @@ parsec_addtest_executable(C dtd_test_interleave_actions SOURCES dtd_test_interle parsec_addtest_executable(C dtd_test_ce SOURCES dtd_test_ce.c) parsec_addtest_executable(C dtd_test_new_tile SOURCES dtd_test_new_tile.c) -if( PARSEC_HAVE_CUDA ) +if( PARSEC_HAVE_LEVEL_ZERO ) + if(NOT TARGET MKL::MKL_DPCPP) + find_package(MKL) + if(NOT MKL_FOUND) + message(WARNING "Could not find the package for MKL. Define the CMake variable MKL_ROOT to point to the directory holding MKLConfig.cmake, or the MKL_DIR environment variable to point to the same directory") + endif(NOT MKL_FOUND) + endif(NOT TARGET MKL::MKL_DPCPP) + if(TARGET MKL::MKL_DPCPP) + message(STATUS "MKL was found and target MKL::MKL_DPCPP is defined. Level-zero can be tested") + parsec_addtest_executable(C dtd_test_simple_gemm_lz SOURCES dtd_test_simple_gemm_lz.c dtd_test_simple_gemm_lz_kernel.dpcpp.C) + target_link_libraries(dtd_test_simple_gemm_lz PRIVATE MKL::MKL_DPCPP level_zero::ze_loader m) + find_package(BLAS) + if(BLAS_FOUND) + target_link_libraries(dtd_test_simple_gemm_lz PRIVATE ${BLAS_LIBRARIES}) + target_include_directories(dtd_test_simple_gemm_lz PRIVATE ${BLAS_INCLUDE_DIRS}) + target_compile_definitions(dtd_test_simple_gemm_lz PRIVATE HAVE_BLAS=1) + endif(BLAS_FOUND) + endif(TARGET MKL::MKL_DPCPP) +elseif( PARSEC_HAVE_CUDA ) parsec_addtest_executable(C dtd_test_cuda_task_insert SOURCES dtd_test_cuda_task_insert.c) if( TARGET CUDA::cublas ) parsec_addtest_executable(C dtd_test_simple_gemm SOURCES dtd_test_simple_gemm.c) @@ -37,5 +55,4 @@ if( PARSEC_HAVE_CUDA ) endif(BLAS_FOUND) target_sources(dtd_test_new_tile PUBLIC dtd_test_new_tile_cuda_kernels.cu) endif( TARGET CUDA::cublas ) -else() -endif() +endif( PARSEC_HAVE_LEVEL_ZERO ) diff --git a/tests/dsl/dtd/dtd_test_simple_gemm_lz.c b/tests/dsl/dtd/dtd_test_simple_gemm_lz.c new file mode 100644 index 000000000..e76deb0e7 --- /dev/null +++ b/tests/dsl/dtd/dtd_test_simple_gemm_lz.c @@ -0,0 +1,564 @@ +#include +#include "parsec.h" +#include "parsec/arena.h" +#include "parsec/data_dist/matrix/matrix.h" +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include "parsec/interfaces/dtd/insert_function_internal.h" + +// The file is not compiled if LEVEL_ZERO is not present or cannot be compiled +#include "parsec/mca/device/cuda/device_cuda.h" + +#if defined(HAVE_BLAS) +// If our CMake finds a BLAS library, it defines HAVE_BLAS +// BLAS does not guarantee there is a cblas.h, we define our own prototype +typedef enum CBLAS_LAYOUT {CblasRowMajor=101, CblasColMajor=102} CBLAS_LAYOUT; +typedef enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113} CBLAS_TRANSPOSE; +typedef enum CBLAS_UPLO {CblasUpper=121, CblasLower=122} CBLAS_UPLO; +typedef enum CBLAS_DIAG {CblasNonUnit=131, CblasUnit=132} CBLAS_DIAG; +typedef enum CBLAS_SIDE {CblasLeft=141, CblasRight=142} CBLAS_SIDE; +#define CBLAS_INDEX int + +extern void cblas_dgemm(const CBLAS_LAYOUT layout, const CBLAS_TRANSPOSE TransA, + const CBLAS_TRANSPOSE TransB, const CBLAS_INDEX M, const CBLAS_INDEX N, + const CBLAS_INDEX K, const double alpha, const double *A, + const CBLAS_INDEX lda, const double *B, const CBLAS_INDEX ldb, + const double beta, double *C, const CBLAS_INDEX ldc); +#endif + +#if defined(PARSEC_HAVE_MPI) +#include +#endif /* defined(PARSEC_HAVE_MPI) */ + +#include +#include + +static int TILE_FULL = -1; +int gemm_lz_verbose = 0; +static int device = PARSEC_DEV_LEVEL_ZERO; +static int P = -1; +static int Q = -1; + +#define Rnd64_A 6364136223846793005ULL +#define Rnd64_C 1ULL +#define RndF_Mul 5.4210108624275222e-20f +#define RndD_Mul 5.4210108624275222e-20 +#define NBELEM 1 + +static unsigned long long int Rnd64_jump(unsigned long long int n, unsigned long long int seed) +{ + unsigned long long int a_k, c_k, ran; + int i; + + a_k = Rnd64_A; + c_k = Rnd64_C; + + ran = seed; + for( i = 0; n; n >>= 1, ++i ) { + if( n & 1 ) + ran = a_k * ran + c_k; + c_k *= (a_k + 1); + a_k *= a_k; + } + + return ran; +} + +int initialize_tile(parsec_execution_stream_t *es, parsec_task_t *this_task) +{ + (void)es; + double *data; + int i, j, mb, nb, m, n, M, ld; + unsigned int seed; + unsigned long long jump, ran; + + parsec_dtd_unpack_args(this_task, &data, &m, &n, &mb, &nb, &M, &ld, &seed); + + jump = (unsigned long long int)m + (unsigned long long int)n * (unsigned long long int)M; + + for( j = 0; j < nb; j++ ) { + ran = Rnd64_jump(NBELEM * jump, seed); + for( i = 0; i < mb; i++ ) { + *data = 0.5f - ran * RndF_Mul; + ran = Rnd64_A * ran + Rnd64_C; + data++; + } + data += ld - i; + jump += M; + } + return PARSEC_HOOK_RETURN_DONE; +} + +int initialize_matrix(parsec_context_t *parsec_context, int rank, parsec_matrix_block_cyclic_t *mat, unsigned int seed, + const char *name, int *gpu_device_index, int nb_gpus) +{ + parsec_taskpool_t *tp = parsec_dtd_taskpool_new(); + + parsec_data_key_t key; + int perr; + + parsec_task_class_t *init_tc; + + perr = parsec_context_start(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_start"); + + // Registering the dtd_handle with PARSEC context + perr = parsec_context_add_taskpool(parsec_context, tp); + PARSEC_CHECK_ERROR(perr, "parsec_context_add_taskpool"); + + init_tc = parsec_dtd_create_task_class(tp, "init", + PASSED_BY_REF, PARSEC_INOUT | TILE_FULL | PARSEC_AFFINITY, + sizeof(int), PARSEC_VALUE, /* m */ + sizeof(int), PARSEC_VALUE, /* n */ + sizeof(int), PARSEC_VALUE, /* mb */ + sizeof(int), PARSEC_VALUE, /* nb */ + sizeof(int), PARSEC_VALUE, /* M */ + sizeof(int), PARSEC_VALUE, /* ld */ + sizeof(unsigned int), PARSEC_VALUE, /* seed */ + PARSEC_DTD_ARG_END); + parsec_dtd_task_class_add_chore(tp, init_tc, PARSEC_DEV_CPU, initialize_tile); + + int g = 0; + for( int i = 0; i < mat->super.mt; i++ ) { + for( int j = 0; j < mat->super.nt; j++ ) { + key = mat->super.super.data_key(&mat->super.super, i, j); + parsec_dtd_insert_task_with_task_class(tp, init_tc, 1, PARSEC_DEV_CPU, + PARSEC_PUSHOUT, PARSEC_DTD_TILE_OF_KEY(&mat->super.super, key), + PARSEC_DTD_EMPTY_FLAG, &i, + PARSEC_DTD_EMPTY_FLAG, &j, + PARSEC_DTD_EMPTY_FLAG, &mat->super.mb, + PARSEC_DTD_EMPTY_FLAG, &mat->super.nb, + PARSEC_DTD_EMPTY_FLAG, &mat->super.m, + PARSEC_DTD_EMPTY_FLAG, &mat->super.mb, + PARSEC_DTD_EMPTY_FLAG, &seed, + PARSEC_DTD_ARG_END); + if(PARSEC_DEV_CUDA == device && + (int)mat->super.super.rank_of_key(&mat->super.super, key) == rank ) { + if( gemm_lz_verbose ) { + fprintf(stderr, "Advice %s(%d, %d) to prefer GPU device %d (parsec device %d) of rank %d\n", + name, i, j, g, gpu_device_index[g], (int)mat->super.super.rank_of_key(&mat->super.super, key)); + } + parsec_advise_data_on_device(mat->super.super.data_of_key(&mat->super.super, key), + gpu_device_index[g], + PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE); + } + g = (g + 1) % nb_gpus; + } + } + parsec_dtd_data_flush_all(tp, &mat->super.super); + + // Wait for task completion + perr = parsec_dtd_taskpool_wait(tp); + PARSEC_CHECK_ERROR(perr, "parsec_dtd_taskpool_wait"); + + perr = parsec_context_wait(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_wait"); + + parsec_dtd_task_class_release(tp, init_tc); + + parsec_taskpool_free(tp); + + return 0; +} + +int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream); + +#if defined(HAVE_BLAS) +int gemm_kernel_cpu(parsec_execution_stream_t *es, + parsec_task_t *this_task) +{ + double *A, *B, *C; + int m, n, k, mb, nb, kb; + double alpha = 1.0; + double beta = 1.0; + double delta; + struct timeval start, end, diff; + + (void)es; + + parsec_dtd_unpack_args(this_task, + &A, &B, &C, + &m, &n, &k, + &mb, &nb, &kb); + + gettimeofday(&start, NULL); + cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, mb, nb, kb, alpha, A, mb, B, kb, beta, C, mb); + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + + delta = (double)diff.tv_sec + (double)diff.tv_usec/1e6; + if( gemm_lz_verbose ) + fprintf(stderr, "GEMM(%d, %d, %d) with tiles of %dx%d, %dx%d, %dx%d on node %d, on core %d: %g s\n", + m, n, k, mb, kb, kb, nb, mb, kb, + this_task->taskpool->context->my_rank, + es->core_id, + delta); + + return PARSEC_HOOK_RETURN_DONE; +} +#endif + +int simple_gemm(parsec_context_t *parsec_context, parsec_matrix_block_cyclic_t *A, parsec_matrix_block_cyclic_t *B, parsec_matrix_block_cyclic_t *C) +{ + parsec_taskpool_t *tp = parsec_dtd_taskpool_new(); + + parsec_data_key_t keyA, keyB, keyC; + int perr; + + parsec_task_class_t *gemm_tc; + + perr = parsec_context_start(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_start"); + + // Registering the dtd_handle with PARSEC context + perr = parsec_context_add_taskpool(parsec_context, tp); + PARSEC_CHECK_ERROR(perr, "parsec_context_add_taskpool"); + + gemm_tc = parsec_dtd_create_task_class(tp, "GEMM", + PASSED_BY_REF, PARSEC_INPUT | TILE_FULL, /* A */ + PASSED_BY_REF, PARSEC_INPUT | TILE_FULL, /* B */ + PASSED_BY_REF, PARSEC_INOUT | TILE_FULL | PARSEC_AFFINITY, /* C */ + sizeof(int), PARSEC_VALUE, /* m */ + sizeof(int), PARSEC_VALUE, /* n */ + sizeof(int), PARSEC_VALUE, /* k */ + sizeof(int), PARSEC_VALUE, /* mb */ + sizeof(int), PARSEC_VALUE, /* nb */ + sizeof(int), PARSEC_VALUE, /* kb */ + PARSEC_DTD_ARG_END); + parsec_dtd_task_class_add_chore(tp, gemm_tc, PARSEC_DEV_LEVEL_ZERO, gemm_kernel_lz); +#if defined(HAVE_BLAS) + parsec_dtd_task_class_add_chore(tp, gemm_tc, PARSEC_DEV_CPU, gemm_kernel_cpu); +#endif + + for( int i = 0; i < C->super.mt; i++ ) { + for( int j = 0; j < C->super.nt; j++ ) { + keyC = C->super.super.data_key(&C->super.super, i, j); + for( int k = 0; k < A->super.nt; k++ ) { + keyA = A->super.super.data_key(&A->super.super, i, k); + keyB = B->super.super.data_key(&B->super.super, k, j); + parsec_dtd_insert_task_with_task_class(tp, gemm_tc, C->super.mt*C->super.nt*A->super.nt - i*C->super.nt + j, device, + PARSEC_INPUT, PARSEC_DTD_TILE_OF_KEY(&A->super.super, keyA), + PARSEC_INPUT, PARSEC_DTD_TILE_OF_KEY(&B->super.super, keyB), + k == A->super.nt - 1 ? (PARSEC_INOUT | PARSEC_PUSHOUT) : PARSEC_INOUT, + PARSEC_DTD_TILE_OF_KEY(&C->super.super, keyC), + PARSEC_DTD_EMPTY_FLAG, &i, + PARSEC_DTD_EMPTY_FLAG, &j, + PARSEC_DTD_EMPTY_FLAG, &k, + PARSEC_DTD_EMPTY_FLAG, &C->super.mb, + PARSEC_DTD_EMPTY_FLAG, &C->super.nb, + PARSEC_DTD_EMPTY_FLAG, &B->super.mb, + PARSEC_DTD_ARG_END); + } + } + } + parsec_dtd_data_flush_all(tp, &A->super.super); + parsec_dtd_data_flush_all(tp, &B->super.super); + parsec_dtd_data_flush_all(tp, &C->super.super); + + // Wait for task completion + perr = parsec_dtd_taskpool_wait(tp); + PARSEC_CHECK_ERROR(perr, "parsec_dtd_taskpool_wait"); + + perr = parsec_context_wait(parsec_context); + PARSEC_CHECK_ERROR(perr, "parsec_context_wait"); + + parsec_dtd_task_class_release(tp, gemm_tc); + + parsec_taskpool_free(tp); + + return 0; +} + +int get_nb_gpu_devices() +{ + int nb = 0; + + for( int dev = 0; dev < (int)parsec_nb_devices; dev++ ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( PARSEC_DEV_LEVEL_ZERO == device->type ) { + nb++; + } + } + + return nb; +} + +int *get_gpu_device_index() +{ + int *dev_index = NULL; + + dev_index = (int *)malloc(parsec_nb_devices * sizeof(int)); + int i = 0; + for( int dev = 0; dev < (int)parsec_nb_devices; dev++ ) { + parsec_device_module_t *device = parsec_mca_device_get(dev); + if( PARSEC_DEV_LEVEL_ZERO == device->type ) { + dev_index[i++] = device->device_index; + } + } + + return dev_index; +} + +static parsec_matrix_block_cyclic_t *create_initialize_matrix(parsec_context_t *parsec_context, int rank, unsigned int seed, const char *name, int mb, int nb, int M, int N, int *gpu_device_index, int nbgpus) +{ + parsec_matrix_block_cyclic_t *dc; + dc = calloc(1, sizeof(parsec_matrix_block_cyclic_t)); + parsec_matrix_block_cyclic_init(dc, PARSEC_MATRIX_DOUBLE, PARSEC_MATRIX_TILE, rank, + mb, nb, + M, N, + 0, 0, + M, N, + P, Q, + 1, 1, + 0, 0); + parsec_data_collection_t *A = &dc->super.super; + parsec_data_collection_set_key(A, name); + dc->mat = parsec_data_allocate((size_t)dc->super.nb_local_tiles * + (size_t)dc->super.bsiz * + (size_t)parsec_datadist_getsizeoftype(dc->super.mtype)); + parsec_dtd_data_collection_init(A); + initialize_matrix(parsec_context, rank, dc, seed, name, gpu_device_index, nbgpus); + + return dc; +} + +static void destroy_matrix(parsec_matrix_block_cyclic_t *dc) +{ + parsec_data_collection_t *A = &dc->super.super; + parsec_dtd_data_collection_fini(A); + if( NULL != dc->mat ) { + parsec_data_free(dc->mat); + } + parsec_tiled_matrix_destroy_data(&dc->super); + parsec_data_collection_destroy(&dc->super.super); + + free(dc); +} + +int main(int argc, char **argv) +{ + int ret = 0, rc, nbgpus = 0; + parsec_context_t *parsec_context = NULL; + int rank, world; + int mb = 1024, nb = 1024, kb = 1024; + int M = 16 * mb, N = 16 * nb, K = 16 * kb; + double min_perf=0.0; + int runs = 5; + int debug=-1; + +#if defined(PARSEC_HAVE_MPI) + { + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &provided); + } + MPI_Comm_size(MPI_COMM_WORLD, &world); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); +#else + world = 1; + rank = 0; +#endif + + while( 1 ) { + int option_index = 0; + static struct option long_options[] = { + {"M", required_argument, 0, 'M'}, + {"N", required_argument, 0, 'N'}, + {"K", required_argument, 0, 'K'}, + {"mb", required_argument, 0, 'm'}, + {"nb", required_argument, 0, 'n'}, + {"kb", required_argument, 0, 'k'}, + {"P", required_argument, 0, 'P'}, + {"Q", required_argument, 0, 'Q'}, + {"device", required_argument, 0, 'd'}, + {"nruns", required_argument, 0, 't'}, + {"verbose", no_argument, 0, 'v'}, + {"Debug", required_argument, 0, 'D'}, + {"Alarm", required_argument, 0, 'A'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + int c = getopt_long(argc, argv, "M:N:K:m:n:k:P:Q:t:d:D:A:vh", + long_options, &option_index); + if( c == -1 ) + break; + + switch( c ) { + case 'M': + M = atoi(optarg); + break; + case 'N': + N = atoi(optarg); + break; + case 'K': + K = atoi(optarg); + break; + case 'm': + mb = atoi(optarg); + break; + case 'n': + nb = atoi(optarg); + break; + case 'k': + kb = atoi(optarg); + break; + case 'P': + P = atoi(optarg); + break; + case 'Q': + Q = atoi(optarg); + break; + case 't': + runs = atoi(optarg); + break; + case 'v': + gemm_lz_verbose = !gemm_lz_verbose; + break; + case 'd': + if(strcmp(optarg, "GPU") == 0) { + device=PARSEC_DEV_CUDA; + } else if(strcmp(optarg, "CPU") == 0) { +#if defined(HAVE_BLAS) + device=PARSEC_DEV_CPU; +#else + fprintf(stderr, "Error: requested to run on CPU (--device=CPU), but no BLAS library has been found at configure time\n"); + exit(1); +#endif + } else { + fprintf(stderr, "Error: device parameter should either be 'GPU' or 'CPU' (got '%s')\n", optarg); + exit(1); + } + break; + case 'D': + debug = atoi(optarg); + break; + case 'A': + min_perf = strtod(optarg, NULL); + break; + case 'h': + case '?': + fprintf(stderr, + "Usage %s [flags] [-- ]\n" + " Nota Bene: this test should not be used to evaluate performance of GEMM!\n" + " Use DPLASMA or other linear algebra libraries written on top of PaRSEC to evaluate this.\n" + "\n" + " Compute pdgemm on a process grid of PxQ, using all available GPUs on each\n" + " node (modulo parsec options), using DTD. Compute C += AxB, where A is MxK\n" + " tiled in mb x kb, B is KxN tiled in kb x nb, and C is MxN tiled in mb x nb\n" + " Executes nruns iterations of the GEMM operation.\n" + " flags:\n" + " --M|-M / --K|-K / --N|-N: set M, K and N (resp.)\n" + " --mb|-m / --kb/-k / --nb|-n: set mb, kb and nb (resp.)\n" + " --nruns|-t: set the number of runs to do\n" + " --device|-d: which device to use (CPU or GPU)\n" + " --verbose|-v: display which GEMM runs on which GPU\n" + " as execution is unfolding\n" + " --help|-h|-?: display this help\n" + " --debug|-D: blocks the process passed as parameter and\n" + " waits for gdb to connect to it\n" + " --Alarm|-A: sets the expected minimum performance for a\n" + " single GPU (kills the process if it takes longer\n" + " than the time corresponding to the expected\n" + " performance to complete the product)\n" + "\n" + " Nota Bene: this test should not be used to evaluate performance of GEMM!\n" + " Use DPLASMA or other linear algebra libraries written on top of PaRSEC to evaluate this.\n" + "\n", + argv[0]); + break; + } + } + int pargc = argc - optind + 1; + char **pargv = (char **)malloc((pargc + 1) * sizeof(char *)); + pargv[0] = argv[0]; + for( int i = 0; i < argc - optind; i++ ) + pargv[i + 1] = argv[optind + i]; + pargv[pargc] = NULL; + + if( -1 == P ) + P = (int)sqrt(world); + if( -1 == Q ) + Q = world / P; + while( P * Q != world ) { + P--; + Q = world / P; + } + + if(debug == rank) { + int loop=1; + char hostname[64]; + gethostname(hostname, 64); + fprintf(stderr, "ssh -t %s gdb -p %d\n", hostname, getpid()); + while(loop) { sleep(1); } + } + + // Number of CPU cores involved + int ncores = -1; // Use all available cores + parsec_context = parsec_init(ncores, &pargc, &pargv); + + int *gpu_device_index = NULL; + if( PARSEC_DEV_LEVEL_ZERO == device ) { + nbgpus = get_nb_gpu_devices(); + rc = !(nbgpus >= 1); + if( rc != 0 ) { + fprintf(stderr, "Rank %d doesn't have CUDA accelerators\n", rank); + MPI_Abort(MPI_COMM_WORLD, 0); + return -1; + } + gpu_device_index = get_gpu_device_index(); + } + + // Create datatypes + parsec_arena_datatype_t *adt = parsec_dtd_create_arena_datatype(parsec_context, &TILE_FULL); + parsec_add2arena_rect(adt, parsec_datatype_double_t, mb, nb, mb); + + // Create and initialize the data + parsec_matrix_block_cyclic_t *dcA = create_initialize_matrix(parsec_context, rank, 1789, "A", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcB = create_initialize_matrix(parsec_context, rank, 1805, "B", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcC = create_initialize_matrix(parsec_context, rank, 1901, "C", mb, nb, M, N, + gpu_device_index, nbgpus); + + for( int r = 0; r < runs + 1; r++ ) { + double gflop = 2.0 * M * N * K / 1e9; + double maxtime = 0.0; + if(min_perf > 0.0) + maxtime = gflop/world/nbgpus/min_perf; + struct timeval start, end, diff; + if(maxtime > 0.0 && maxtime < 60.0) maxtime=60.0; + if(rank == 0 && maxtime > 0.0) fprintf(stderr, "watchdog: %d seconds\n", (int)maxtime); + if(maxtime > 0.0) alarm((int)maxtime); + gettimeofday(&start, NULL); + simple_gemm(parsec_context, dcA, dcB, dcC); + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + double t = (double)diff.tv_sec + (double)diff.tv_usec / 1e6; + double gflops = gflop / t; + (void)t; + (void)gflops; + if( 0 == rank && r > 0 ) { + fprintf(stderr, "DTD_GEMM PxQxg: %d %d %d M: %d N: %d K: %d mb: %d nb: %d kb: %d -- done\n", + P, Q, nbgpus, M, N, K, mb, nb, kb); + } + } + // deactivate the alarm if it was set + alarm(0); + + parsec_type_free(&adt->opaque_dtt); + PARSEC_OBJ_RELEASE(adt->arena); + parsec_dtd_destroy_arena_datatype(parsec_context, TILE_FULL); + + destroy_matrix(dcA); + destroy_matrix(dcB); + destroy_matrix(dcC); + + parsec_fini(&parsec_context); + +#if defined(PARSEC_HAVE_MPI) + MPI_Finalize(); +#endif + + return ret; +} diff --git a/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C b/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C new file mode 100644 index 000000000..80228aed9 --- /dev/null +++ b/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C @@ -0,0 +1,75 @@ +#include "parsec/data_dist/matrix/two_dim_rectangle_cyclic.h" +#include "parsec/mca/device/level_zero/device_level_zero.h" +#include "parsec/interfaces/dtd/insert_function.h" +#include "parsec/runtime.h" +#include "parsec/execution_stream.h" + +#include +#include + +extern "C" { + extern int gemm_lz_verbose; + int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream); +} + +int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, + parsec_gpu_task_t *gpu_task, + parsec_gpu_exec_stream_t *gpu_stream) +{ + double *A, *B, *C; + int m, n, k, mb, nb, kb; + parsec_task_t *this_task = gpu_task->ec; + struct timeval start, end, diff; + double delta; + double *a_gpu, *b_gpu, *c_gpu; + parsec_level_zero_exec_stream_t *lz_stream = (parsec_level_zero_exec_stream_t *)gpu_stream; + + (void)gpu_device; + + parsec_dtd_unpack_args(this_task, + &A, &B, &C, + &m, &n, &k, + &mb, &nb, &kb); + + a_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 0)); + b_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 1)); + c_gpu = static_cast(parsec_dtd_get_dev_ptr(this_task, 2)); + + gettimeofday(&start, NULL); + + double alpha=0.0; + double beta=1.0; + try { + oneapi::mkl::blas::gemm(lz_stream->swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + mb, kb, nb, + alpha, static_cast(a_gpu), mb, + static_cast(b_gpu), kb, + beta, static_cast(c_gpu), nb); + } catch (const oneapi::mkl::invalid_argument &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws invalid argument exception"); + } catch (const oneapi::mkl::unsupported_device &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unsuported device exception"); + } catch (const oneapi::mkl::host_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws host bad allocation exception"); + } catch (const oneapi::mkl::device_bad_alloc &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws device bad allocation exception"); + } catch (const oneapi::mkl::unimplemented &e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unimplemented exception"); + } catch (const std::exception& e) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception"); + } catch (...) { + parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); + } + gettimeofday(&end, NULL); + timersub(&end, &start, &diff); + delta = (double)diff.tv_sec + (double)diff.tv_usec/1e6; + if(gemm_lz_verbose) + fprintf(stderr, "GEMM(%d, %d, %d) with tiles of %dx%d, %dx%d, %dx%d on node %d, GPU %s submitted in %g s\n", + m, n, k, mb, kb, kb, nb, mb, kb, + this_task->taskpool->context->my_rank, + gpu_stream->name, delta); + + return PARSEC_HOOK_RETURN_DONE; +} From 138528983286d75ab80d0d630472ae992bd36f1f Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Fri, 16 Dec 2022 23:29:13 +0000 Subject: [PATCH 10/15] Manage LEVEL_ZERO devices in DTD --- parsec/interfaces/dtd/insert_function.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/parsec/interfaces/dtd/insert_function.c b/parsec/interfaces/dtd/insert_function.c index 3b6867f24..d2f4e717d 100644 --- a/parsec/interfaces/dtd/insert_function.c +++ b/parsec/interfaces/dtd/insert_function.c @@ -41,6 +41,9 @@ #if defined(PARSEC_HAVE_CUDA) #include "parsec/mca/device/cuda/device_cuda.h" #endif /* defined(PARSEC_HAVE_CUDA) */ +#if defined(PARSEC_HAVE_LEVEL_ZERO) +#include "parsec/mca/device/level_zero/device_level_zero.h" +#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */ #include "parsec/mca/mca_repository.h" #include "parsec/constants.h" @@ -1553,6 +1556,9 @@ parsec_dtd_startup(parsec_context_t *context, // If CUDA is enabled, let the CUDA device activated for this // taskpool. if( PARSEC_DEV_CUDA == device->type ) continue; + // If LEVEL_ZERO is enabled, let the LEVEL_ZERO device activated for this + // taskpool. + if( PARSEC_DEV_LEVEL_ZERO == device->type ) continue; if( NULL != device->taskpool_register ) if( PARSEC_SUCCESS != device->taskpool_register(device, (parsec_taskpool_t *)tp)) { @@ -2396,6 +2402,12 @@ static parsec_hook_return_t parsec_dtd_gpu_task_submit(parsec_execution_stream_t gpu_task->stage_in = parsec_default_cuda_stage_in; gpu_task->stage_out = parsec_default_cuda_stage_out; return parsec_cuda_kernel_scheduler(es, gpu_task, dev_index); +#endif +#if defined(PARSEC_HAVE_LEVEL_ZERO) + case PARSEC_DEV_LEVEL_ZERO: + gpu_task->stage_in = parsec_default_level_zero_stage_in; + gpu_task->stage_out = parsec_default_level_zero_stage_out; + return parsec_level_zero_kernel_scheduler(es, gpu_task, dev_index); #endif default: parsec_fatal("DTD scheduling on device type %d: this is not a valid GPU device type in this build", device->type); @@ -2466,8 +2478,10 @@ int parsec_dtd_task_class_add_chore(parsec_taskpool_t *tp, if(PARSEC_DEV_CUDA == device_type) { incarnations[i].hook = parsec_dtd_gpu_task_submit; dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)function; - } - else { + } else if(PARSEC_DEV_LEVEL_ZERO == device_type) { + incarnations[i].hook = parsec_dtd_gpu_task_submit; + dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)function; + } else { dtd_tc->cpu_func_ptr = function; incarnations[i].hook = parsec_dtd_cpu_task_submit; } @@ -3351,8 +3365,11 @@ __parsec_dtd_taskpool_create_task(parsec_taskpool_t *tp, /* Special case for CUDA: we need an intermediate */ (*incarnations)[0].hook = parsec_dtd_gpu_task_submit; dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)fpointer; - } - else { + } if( device_type == PARSEC_DEV_LEVEL_ZERO ) { + /* Special case for LEVEL ZERO: we need an intermediate */ + (*incarnations)[0].hook = parsec_dtd_gpu_task_submit; + dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)fpointer; + } else { /* Default case: the user-provided function is directly the hook to call */ (*incarnations)[0].hook = fpointer; // We can directly call the CPU hook dtd_tc->cpu_func_ptr = fpointer; From 486427409fd72bd2520a9347b1f8cec6151ef797 Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Fri, 16 Dec 2022 23:29:39 +0000 Subject: [PATCH 11/15] Accept LEVEL_ZERO devices in the PTG generated code Some fixes in device level_zero Temp fix for termination detection -- tag size must be made portable. TODO! --- parsec/interfaces/ptg/ptg-compiler/jdf2c.c | 1 + .../mca/device/level_zero/device_level_zero.h | 2 +- .../level_zero/device_level_zero_module.c | 50 ++++++++++--------- .../termdet/fourcounter/termdet_fourcounter.h | 2 +- .../fourcounter/termdet_fourcounter_module.c | 2 +- .../user_trigger/termdet_user_trigger.h | 2 +- .../termdet_user_trigger_module.c | 2 +- 7 files changed, 32 insertions(+), 29 deletions(-) diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index dc5bc25a0..338a6e439 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -6779,6 +6779,7 @@ static void jdf_generate_code_hook_cuda(const jdf_t *jdf, " int chore_idx = 0;\n" " for ( ; PARSEC_DEV_NONE != this_task->task_class->incarnations[chore_idx].type; ++chore_idx) {\n" " if (this_task->task_class->incarnations[chore_idx].type == PARSEC_DEV_CUDA) break;\n" + " if (this_task->task_class->incarnations[chore_idx].type == PARSEC_DEV_LEVEL_ZERO) break;\n" " }\n" " /* The void* cast prevents the compiler from complaining about the type change */\n" " parsec_body.dyld_fn = (%s)(void*)this_task->task_class->incarnations[chore_idx].dyld_fn;\n" diff --git a/parsec/mca/device/level_zero/device_level_zero.h b/parsec/mca/device/level_zero/device_level_zero.h index e7f06ab61..4505f7fd9 100644 --- a/parsec/mca/device/level_zero/device_level_zero.h +++ b/parsec/mca/device/level_zero/device_level_zero.h @@ -63,8 +63,8 @@ struct parsec_level_zero_exec_stream_s { * remains in the system the function is supposed to update it. */ ze_event_handle_t *events; + ze_command_list_handle_t *command_lists; ze_event_pool_handle_t ze_event_pool; - ze_command_list_handle_t level_zero_cl; ze_command_queue_handle_t level_zero_cq; parsec_sycl_wrapper_queue_t *swq; }; diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c index 3f014fe3a..1f06158bf 100644 --- a/parsec/mca/device/level_zero/device_level_zero_module.c +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -247,15 +247,6 @@ int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t ze_rc = zeCommandQueueCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, &commandQueueDesc, &level_zero_stream->level_zero_cq); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, {goto release_device;} ); - ze_command_list_desc_t commandListDesc = { - ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, - NULL, - computeQueueGroupOrdinal, - 0 // flags - }; - ze_rc = zeCommandListCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, - &commandListDesc, &level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, {goto release_device;} ); exec_stream->workspace = NULL; PARSEC_OBJ_CONSTRUCT(&exec_stream->infos, parsec_info_object_array_t); parsec_info_object_array_init(&exec_stream->infos, &parsec_per_stream_infos, exec_stream); @@ -269,6 +260,7 @@ int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t exec_stream->tasks = (parsec_gpu_task_t**)malloc(exec_stream->max_events * sizeof(parsec_gpu_task_t*)); level_zero_stream->events = (ze_event_handle_t*)malloc(exec_stream->max_events * sizeof(ze_event_handle_t)); + level_zero_stream->command_lists = (ze_command_list_handle_t*)malloc(exec_stream->max_events*sizeof(ze_command_list_handle_t)); // Create event pool ze_event_pool_desc_t eventPoolDesc = { @@ -282,6 +274,12 @@ int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t /* and the corresponding events */ for( k = 0; k < exec_stream->max_events; k++ ) { + ze_command_list_desc_t commandListDesc = { + ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, + NULL, + computeQueueGroupOrdinal, + 0 // flags + }; ze_event_desc_t eventDesc = { ZE_STRUCTURE_TYPE_EVENT_DESC, NULL, @@ -291,9 +289,11 @@ int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t }; level_zero_stream->events[k] = NULL; exec_stream->tasks[k] = NULL; + ze_rc = zeCommandListCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, + &commandListDesc, &level_zero_stream->command_lists[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, {goto release_device;} ); ze_rc = zeEventCreate(level_zero_stream->ze_event_pool, &eventDesc, &(level_zero_stream->events[k])); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventCreate ", ze_rc, - {goto release_device;} ); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventCreate ", ze_rc, {goto release_device;} ); } if(j == 0) { len = asprintf(&exec_stream->name, "h2d(%d)", j); @@ -460,6 +460,8 @@ parsec_level_zero_module_fini(parsec_device_module_t* device) assert( NULL == exec_stream->tasks[k] ); status = zeEventDestroy(level_zero_stream->events[k]); PARSEC_LEVEL_ZERO_CHECK_ERROR( "(parsec_level_zero_device_fini) level_zeroEventDestroy ", status, {} ); + status = zeCommandListDestroy(level_zero_stream->command_lists[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListDestroy ", status, {} ); } zeEventPoolDestroy(level_zero_stream->ze_event_pool); exec_stream->max_events = 0; @@ -468,8 +470,6 @@ parsec_level_zero_module_fini(parsec_device_module_t* device) free(exec_stream->fifo_pending); exec_stream->fifo_pending = NULL; /* Release the stream */ ze_result_t ze_rc; - ze_rc = zeCommandListDestroy(level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListDestroy ", ze_rc, {} ); /* Deleting the sycl queue wrapper and/or the command queue conflicts with the * cleaning procedure of the Level Zero runtime... Didn't find a way to do it * cleanly. Don't cleanup for now... */ @@ -550,10 +550,9 @@ parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* level_zero_ } initial_free_mem = devProperties.maxMemAllocSize < devMemProperties[memIndex].totalSize ? devProperties.maxMemAllocSize : devMemProperties[memIndex].totalSize; - PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, - "level-zero device %s: initial_free_mem is %lu (device max alloc size is %lu, device memory " - "property total size for memory bank %d is %lu)", gpu_device->super.name, initial_free_mem, - devProperties.maxMemAllocSize, memIndex, devMemProperties[memIndex].totalSize); + fprintf(stderr, "level-zero device %s: initial_free_mem is %lu (device max alloc size is %lu, device memory " + "property total size for memory bank %d is %lu)", gpu_device->super.name, initial_free_mem, + devProperties.maxMemAllocSize, memIndex, devMemProperties[memIndex].totalSize); free(devMemProperties); devMemProperties = NULL; if( number_blocks != -1 ) { @@ -1118,7 +1117,7 @@ parsec_default_level_zero_stage_in(parsec_gpu_task_t *gtask, copy_out = task->data[i].data_out; count = (copy_in->original->nb_elts <= copy_out->original->nb_elts) ? copy_in->original->nb_elts : copy_out->original->nb_elts; - ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, + ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[gpu_stream->start], copy_out->device_private, copy_in->device_private, count, @@ -1157,7 +1156,7 @@ parsec_default_level_zero_stage_out(parsec_gpu_task_t *gtask, copy_out = copy_in->original->device_copies[0]; count = (copy_in->original->nb_elts <= copy_out->original->nb_elts) ? copy_in->original->nb_elts : copy_out->original->nb_elts; - ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, + ret = (ze_result_t)zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[gpu_stream->start], copy_out->device_private, copy_in->device_private, count, @@ -1897,6 +1896,8 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, parsec_task_snprintf(task_str, MAX_TASK_STRLEN, task->ec), task->ec->priority, stream->name, (void*)stream); stream->tasks[stream->end] = NULL; + rc = zeCommandListReset(level_zero_stream->command_lists[stream->end]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", rc, {} ); stream->end = (stream->end + 1) % stream->max_events; #if defined(PARSEC_PROF_TRACE) @@ -1977,14 +1978,15 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, * too early, it might get executed before the data is available on the GPU. * Obviously, this lead to incorrect results. */ - rc = zeCommandListAppendSignalEvent( level_zero_stream->level_zero_cl, level_zero_stream->events[stream->start] ); + rc = zeCommandListAppendSignalEvent( level_zero_stream->command_lists[stream->start], level_zero_stream->events[stream->start] ); assert(ZE_RESULT_SUCCESS == rc); - ze_result_t ze_rc = zeCommandListClose(level_zero_stream->level_zero_cl); + ze_result_t ze_rc = zeCommandListClose(level_zero_stream->command_lists[stream->start]); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { } ); - ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->level_zero_cl, NULL); + ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->command_lists[stream->start], NULL); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { } ); - ze_rc = zeCommandListReset(level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { } ); + + ze_rc = zeCommandQueueSynchronize( level_zero_stream->level_zero_cq, 5000000 ); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueSynchronize ", ze_rc, { } ); stream->tasks[stream->start] = task; stream->start = (stream->start + 1) % stream->max_events; diff --git a/parsec/mca/termdet/fourcounter/termdet_fourcounter.h b/parsec/mca/termdet/fourcounter/termdet_fourcounter.h index a7850f22a..7848d79bd 100644 --- a/parsec/mca/termdet/fourcounter/termdet_fourcounter.h +++ b/parsec/mca/termdet/fourcounter/termdet_fourcounter.h @@ -35,7 +35,7 @@ BEGIN_C_DECLS PARSEC_DECLSPEC extern const parsec_termdet_base_component_t parsec_termdet_fourcounter_component; PARSEC_DECLSPEC extern const parsec_termdet_module_t parsec_termdet_fourcounter_module; -int parsec_termdet_fourcounter_msg_dispatch(parsec_comm_engine_t *ce, unsigned long long tag, void *msg, unsigned long size, int src, void *module); +int parsec_termdet_fourcounter_msg_dispatch(parsec_comm_engine_t *ce, unsigned long tag, void *msg, unsigned long size, int src, void *module); typedef enum { PARSEC_TERMDET_FOURCOUNTER_MSG_TYPE_DOWN, diff --git a/parsec/mca/termdet/fourcounter/termdet_fourcounter_module.c b/parsec/mca/termdet/fourcounter/termdet_fourcounter_module.c index 90a98d4d6..4936e7267 100644 --- a/parsec/mca/termdet/fourcounter/termdet_fourcounter_module.c +++ b/parsec/mca/termdet/fourcounter/termdet_fourcounter_module.c @@ -147,7 +147,7 @@ static int parsec_termdet_fourcounter_msg_dispatch_taskpool(parsec_taskpool_t *t return PARSEC_ERROR; } -int parsec_termdet_fourcounter_msg_dispatch(parsec_comm_engine_t *ce, unsigned long long tag, void *msg, unsigned long size, int src, void *module) +int parsec_termdet_fourcounter_msg_dispatch(parsec_comm_engine_t *ce, unsigned long tag, void *msg, unsigned long size, int src, void *module) { parsec_termdet_fourcounter_delayed_msg_t *delayed_msg; parsec_termdet_fourcounter_msg_down_t *down_msg = (parsec_termdet_fourcounter_msg_down_t*)msg; diff --git a/parsec/mca/termdet/user_trigger/termdet_user_trigger.h b/parsec/mca/termdet/user_trigger/termdet_user_trigger.h index 33f5cda88..3db586165 100644 --- a/parsec/mca/termdet/user_trigger/termdet_user_trigger.h +++ b/parsec/mca/termdet/user_trigger/termdet_user_trigger.h @@ -39,7 +39,7 @@ PARSEC_DECLSPEC extern const parsec_termdet_module_t parsec_termdet_user_trigger /* static accessor */ mca_base_component_t *termdet_user_trigger_static_component(void); -int parsec_termdet_user_trigger_msg_dispatch(parsec_comm_engine_t *ce, unsigned long long tag, void *msg, +int parsec_termdet_user_trigger_msg_dispatch(parsec_comm_engine_t *ce, unsigned long tag, void *msg, unsigned long size, int src, void *module); typedef struct { diff --git a/parsec/mca/termdet/user_trigger/termdet_user_trigger_module.c b/parsec/mca/termdet/user_trigger/termdet_user_trigger_module.c index 990457318..d9bf353e4 100644 --- a/parsec/mca/termdet/user_trigger/termdet_user_trigger_module.c +++ b/parsec/mca/termdet/user_trigger/termdet_user_trigger_module.c @@ -114,7 +114,7 @@ static int parsec_termdet_user_trigger_msg_dispatch_taskpool(parsec_taskpool_t * return PARSEC_SUCCESS; } -int parsec_termdet_user_trigger_msg_dispatch(parsec_comm_engine_t *ce, unsigned long long tag, void *msg, +int parsec_termdet_user_trigger_msg_dispatch(parsec_comm_engine_t *ce, unsigned long tag, void *msg, unsigned long size, int src, void *module) { parsec_termdet_user_trigger_delayed_msg_t *delayed_msg; From d5b783030e7e35925ab6bdb2811425b8217d2b8b Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Fri, 16 Dec 2022 23:31:35 +0000 Subject: [PATCH 12/15] Support LEVEL_ZERO devices in the DSL tests Fix the subsystem test. Need to backport fixes in the MCA device Fully functional sketch for level zero --- tests/dsl/dtd/dtd_test_simple_gemm_lz.c | 94 ++++++++- .../dtd_test_simple_gemm_lz_kernel.dpcpp.C | 10 +- tests/dsl/ptg/cuda/nvlink_wrapper.c | 15 +- tests/subsystem/level_zero/main.c | 189 ++++++++++++++---- 4 files changed, 247 insertions(+), 61 deletions(-) diff --git a/tests/dsl/dtd/dtd_test_simple_gemm_lz.c b/tests/dsl/dtd/dtd_test_simple_gemm_lz.c index e76deb0e7..50373fc8c 100644 --- a/tests/dsl/dtd/dtd_test_simple_gemm_lz.c +++ b/tests/dsl/dtd/dtd_test_simple_gemm_lz.c @@ -44,6 +44,8 @@ static int Q = -1; #define RndD_Mul 5.4210108624275222e-20 #define NBELEM 1 +#define EPSILON 1e-10 + static unsigned long long int Rnd64_jump(unsigned long long int n, unsigned long long int seed) { unsigned long long int a_k, c_k, ran; @@ -131,7 +133,7 @@ int initialize_matrix(parsec_context_t *parsec_context, int rank, parsec_matrix_ PARSEC_DTD_EMPTY_FLAG, &mat->super.mb, PARSEC_DTD_EMPTY_FLAG, &seed, PARSEC_DTD_ARG_END); - if(PARSEC_DEV_CUDA == device && + if(PARSEC_DEV_LEVEL_ZERO == device && (int)mat->super.super.rank_of_key(&mat->super.super, key) == rank ) { if( gemm_lz_verbose ) { fprintf(stderr, "Advice %s(%d, %d) to prefer GPU device %d (parsec device %d) of rank %d\n", @@ -140,8 +142,8 @@ int initialize_matrix(parsec_context_t *parsec_context, int rank, parsec_matrix_ parsec_advise_data_on_device(mat->super.super.data_of_key(&mat->super.super, key), gpu_device_index[g], PARSEC_DEV_DATA_ADVICE_PREFERRED_DEVICE); + g = (g + 1) % nb_gpus; } - g = (g + 1) % nb_gpus; } } parsec_dtd_data_flush_all(tp, &mat->super.super); @@ -336,6 +338,25 @@ static void destroy_matrix(parsec_matrix_block_cyclic_t *dc) free(dc); } +static void print_matrix(parsec_matrix_block_cyclic_t *dc, parsec_context_t *parsec_context, const char *info) +{ + for( int i = 0; i < dc->super.mt; i++ ) { + for( int j = 0; j < dc->super.nt; j++ ) { + if( (int)dc->super.super.rank_of(&dc->super.super, i, j) == parsec_context->my_rank ) { + fprintf(stderr, "%-5s(%2d, %2d): ", info, i, j); + parsec_data_t *tile = dc->super.super.data_of(&dc->super.super, i, j); + double *mat = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(tile, 0)); + for( int ii = 0; ii < dc->super.mb; ii++) { + for(int jj = 0; jj < dc->super.nb; jj++) { + fprintf(stderr, "%5.2g ", mat[ii*dc->super.nb + jj]); + } + fprintf(stderr, "\n "); + } + } + } + } +} + int main(int argc, char **argv) { int ret = 0, rc, nbgpus = 0; @@ -346,6 +367,7 @@ int main(int argc, char **argv) double min_perf=0.0; int runs = 5; int debug=-1; + int check = 0; #if defined(PARSEC_HAVE_MPI) { @@ -375,11 +397,12 @@ int main(int argc, char **argv) {"verbose", no_argument, 0, 'v'}, {"Debug", required_argument, 0, 'D'}, {"Alarm", required_argument, 0, 'A'}, + {"Check", no_argument, 0, 'x'}, {"help", no_argument, 0, 'h'}, {0, 0, 0, 0} }; - int c = getopt_long(argc, argv, "M:N:K:m:n:k:P:Q:t:d:D:A:vh", + int c = getopt_long(argc, argv, "M:N:K:m:n:k:P:Q:t:d:D:A:xvh", long_options, &option_index); if( c == -1 ) break; @@ -417,7 +440,7 @@ int main(int argc, char **argv) break; case 'd': if(strcmp(optarg, "GPU") == 0) { - device=PARSEC_DEV_CUDA; + device=PARSEC_DEV_LEVEL_ZERO; } else if(strcmp(optarg, "CPU") == 0) { #if defined(HAVE_BLAS) device=PARSEC_DEV_CPU; @@ -436,6 +459,14 @@ int main(int argc, char **argv) case 'A': min_perf = strtod(optarg, NULL); break; + case 'x': +#if defined(HAVE_BLAS) + check = 1; + runs=1; +#else + fprintf(stderr, "Error: requested to run with checks, but CPU BLAS not available\n"); +#endif + break; case 'h': case '?': fprintf(stderr, @@ -461,6 +492,9 @@ int main(int argc, char **argv) " single GPU (kills the process if it takes longer\n" " than the time corresponding to the expected\n" " performance to complete the product)\n" + " --Check|-x: perform numerical check of the solution by comparing\n" + " the output of the GPU computation with the output of the\n" + " CPU BLAS computation\n" "\n" " Nota Bene: this test should not be used to evaluate performance of GEMM!\n" " Use DPLASMA or other linear algebra libraries written on top of PaRSEC to evaluate this.\n" @@ -513,6 +547,53 @@ int main(int argc, char **argv) parsec_arena_datatype_t *adt = parsec_dtd_create_arena_datatype(parsec_context, &TILE_FULL); parsec_add2arena_rect(adt, parsec_datatype_double_t, mb, nb, mb); + if(check) { + fprintf(stderr, "Computing target GEMM\n"); + parsec_matrix_block_cyclic_t *dcA = create_initialize_matrix(parsec_context, rank, 1789, "A", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcB = create_initialize_matrix(parsec_context, rank, 1805, "B", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcC = create_initialize_matrix(parsec_context, rank, 1901, "C", mb, nb, M, N, + gpu_device_index, nbgpus); + print_matrix(dcC, parsec_context, "C0"); + simple_gemm(parsec_context, dcA, dcB, dcC); + print_matrix(dcC, parsec_context, "C1"); + + device = PARSEC_DEV_CPU; + fprintf(stderr, "Computing CPU BLAS GEMM\n"); + parsec_matrix_block_cyclic_t *dcAcheck = create_initialize_matrix(parsec_context, rank, 1789, "Acheck", mb, kb, M, K, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcBcheck = create_initialize_matrix(parsec_context, rank, 1805, "Bcheck", kb, nb, K, N, + gpu_device_index, nbgpus); + parsec_matrix_block_cyclic_t *dcCcheck = create_initialize_matrix(parsec_context, rank, 1901, "Ccheck", mb, nb, M, N, + gpu_device_index, nbgpus); + print_matrix(dcCcheck, parsec_context, "C'0"); + simple_gemm(parsec_context, dcAcheck, dcBcheck, dcCcheck); + print_matrix(dcCcheck, parsec_context, "C'1"); + + for( int i = 0; i < 0*dcC->super.mt; i++ ) { + for( int j = 0; j < dcC->super.nt; j++ ) { + if( (int)dcC->super.super.rank_of(&dcC->super.super, i, j) == parsec_context->my_rank ) { + parsec_data_t *CTile = dcC->super.super.data_of(&dcC->super.super, i, j); + parsec_data_t *DTile = dcCcheck->super.super.data_of(&dcCcheck->super.super, i, j); + double *c = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(CTile, 0)); + double *d = PARSEC_DATA_COPY_GET_PTR(parsec_data_get_copy(DTile, 0)); + for( int x = 0; x < mb*nb; x++) { + if( fabs(c[x] - d[x]) > EPSILON ) { + fprintf(stderr, "Tile (%d, %d), double %d is %g/%g -- differ by %g > %g\n", i, j, x, c[x], d[x], fabs(c[x]-d[x]), EPSILON ); + } + } + } + } + } + + destroy_matrix(dcA); + destroy_matrix(dcB); + destroy_matrix(dcC); + destroy_matrix(dcAcheck); + destroy_matrix(dcBcheck); + destroy_matrix(dcCcheck); + } else { // Create and initialize the data parsec_matrix_block_cyclic_t *dcA = create_initialize_matrix(parsec_context, rank, 1789, "A", mb, kb, M, K, gpu_device_index, nbgpus); @@ -539,8 +620,8 @@ int main(int argc, char **argv) (void)t; (void)gflops; if( 0 == rank && r > 0 ) { - fprintf(stderr, "DTD_GEMM PxQxg: %d %d %d M: %d N: %d K: %d mb: %d nb: %d kb: %d -- done\n", - P, Q, nbgpus, M, N, K, mb, nb, kb); + fprintf(stderr, "DTD_GEMM PxQxg: %d %d %d M: %d N: %d K: %d mb: %d nb: %d kb: %d -- %g s, %g GFLop => %g GFLop/s\n", + P, Q, nbgpus, M, N, K, mb, nb, kb, t, gflop, gflops); } } // deactivate the alarm if it was set @@ -553,6 +634,7 @@ int main(int argc, char **argv) destroy_matrix(dcA); destroy_matrix(dcB); destroy_matrix(dcC); + } parsec_fini(&parsec_context); diff --git a/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C b/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C index 80228aed9..03d162262 100644 --- a/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C +++ b/tests/dsl/dtd/dtd_test_simple_gemm_lz_kernel.dpcpp.C @@ -25,6 +25,7 @@ int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, double delta; double *a_gpu, *b_gpu, *c_gpu; parsec_level_zero_exec_stream_t *lz_stream = (parsec_level_zero_exec_stream_t *)gpu_stream; + cl::sycl::event gemm_event; (void)gpu_device; @@ -42,11 +43,12 @@ int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, double alpha=0.0; double beta=1.0; try { - oneapi::mkl::blas::gemm(lz_stream->swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, + gemm_event = oneapi::mkl::blas::gemm(lz_stream->swq->queue, oneapi::mkl::transpose::N, oneapi::mkl::transpose::N, mb, kb, nb, alpha, static_cast(a_gpu), mb, static_cast(b_gpu), kb, beta, static_cast(c_gpu), nb); + gemm_event.wait(); } catch (const oneapi::mkl::invalid_argument &e) { parsec_warning("OneAPI MKL BLAS GEMM throws invalid argument exception"); } catch (const oneapi::mkl::unsupported_device &e) { @@ -66,10 +68,10 @@ int gemm_kernel_lz(parsec_device_gpu_module_t *gpu_device, timersub(&end, &start, &diff); delta = (double)diff.tv_sec + (double)diff.tv_usec/1e6; if(gemm_lz_verbose) - fprintf(stderr, "GEMM(%d, %d, %d) with tiles of %dx%d, %dx%d, %dx%d on node %d, GPU %s submitted in %g s\n", - m, n, k, mb, kb, kb, nb, mb, kb, + fprintf(stderr, "GEMM(%d, %d, %d) with tiles of %dx%d [%p], %dx%d [%p], %dx%d [%p] on node %d, Stream %s of GPU %s submitted in %g s\n", + m, n, k, mb, kb, a_gpu, kb, nb, b_gpu, mb, kb, c_gpu, this_task->taskpool->context->my_rank, - gpu_stream->name, delta); + gpu_device->super.name, gpu_stream->name, delta); return PARSEC_HOOK_RETURN_DONE; } diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index 40835aeed..7ee2bb9c3 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -249,15 +249,12 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb #elif defined(PARSEC_HAVE_LEVEL_ZERO) parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; - status = zeCommandListAppendMemoryCopy(level_zero_stream->level_zero_cl, gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); + status = zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[0], gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); - status = zeCommandListClose(level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", status, { return NULL; } ); - status = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->level_zero_cl, NULL); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", status, { return NULL; } ); - status = zeCommandListReset(level_zero_stream->level_zero_cl); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", status, { return NULL; } ); - + status = zeCommandListClose(level_zero_stream->command_lists[0]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", status, { return NULL; } ); + status = zeCommandQueueExecuteCommandLists(level_zero_stream->command_lists[0], 1, &level_zero_stream->command_lists[0], NULL); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", status, { return NULL; } ); while(1) { status = zeEventQueryStatus(copySignalEvent); if(status == ZE_RESULT_SUCCESS) @@ -266,6 +263,8 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventHostSynchronize ", status, { break; } ); usleep(1000); } + status = zeCommandListReset(level_zero_stream->command_lists[0]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", status, { return NULL; } ); #else memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); #endif diff --git a/tests/subsystem/level_zero/main.c b/tests/subsystem/level_zero/main.c index e2444fae4..93e0e5844 100644 --- a/tests/subsystem/level_zero/main.c +++ b/tests/subsystem/level_zero/main.c @@ -1,5 +1,11 @@ #include #include +<<<<<<< HEAD +======= +#include +#include +#include +>>>>>>> f5f749053... Fix the subsystem test. Need to backport fixes in the MCA device #include #include "interface.dpcpp.h" @@ -16,8 +22,7 @@ struct device_s; struct stream_s; #define NB_STREAMS 4 -#define MAX_EVENTS 2 -#define N 256 +#define MAX_FENCES 2 typedef struct stream_s { int immediate; @@ -25,14 +30,13 @@ typedef struct stream_s { ze_command_queue_handle_t cq; ze_command_list_handle_t cl; struct device_s *device; - ze_event_handle_t events[MAX_EVENTS]; + ze_fence_handle_t fences[MAX_FENCES]; } stream_t; typedef struct device_s { ze_device_handle_t device; struct driver_s *driver; sycl_wrapper_device_t *swd; - ze_event_pool_handle_t eventPool; stream_t streams[NB_STREAMS]; } device_t; @@ -91,17 +95,6 @@ static int init_device(device_t *device, ze_device_handle_t gpuDevice) return -1; } - // Create event pool - ze_event_pool_desc_t eventPoolDesc = { - ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, - NULL, - ZE_EVENT_POOL_FLAG_HOST_VISIBLE, // all events in pool are visible to Host - 1 // count - }; - ze_rc = zeEventPoolCreate(device->driver->context, &eventPoolDesc, 0, NULL, - &device->eventPool); - LEVEL_ZERO_CHECK_ERROR( "zeEventPoolCreate ", ze_rc, {return -1;} ); - for(int j = 0; j < NB_STREAMS; j++ ) { ze_command_queue_desc_t commandQueueDesc = { ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, @@ -140,17 +133,15 @@ static int init_device(device_t *device, ze_device_handle_t gpuDevice) return -1; } - for(int k = 0; k < MAX_EVENTS; k++ ) { - ze_event_desc_t eventDesc = { - ZE_STRUCTURE_TYPE_EVENT_DESC, - NULL, - 0, // index - 0, // no additional memory/cache coherency required on signal - ZE_EVENT_SCOPE_FLAG_HOST // ensure memory coherency across device and Host after event completes + for(int k = 0; k < MAX_FENCES; k++ ) { + ze_fence_desc_t fence_desc = { + .stype = ZE_STRUCTURE_TYPE_FENCE_DESC, + .pNext = NULL, + .flags = 0 }; - device->streams[j].events[k] = NULL; - ze_rc = zeEventCreate(device->eventPool, &eventDesc, &(device->streams[j].events[k])); - LEVEL_ZERO_CHECK_ERROR( "zeEventCreate ", ze_rc, {return -1;} ); + device->streams[j].fences[k] = NULL; + ze_rc = zeFenceCreate(device->streams[j].cq, &fence_desc, &(device->streams[j].fences[k])); + LEVEL_ZERO_CHECK_ERROR( "zeFenceCreate ", ze_rc, { continue; } ); } } device->device = gpuDevice; @@ -295,6 +286,35 @@ static void *allocate_workspace(device_t *device, size_t size) return device_ptr; } +typedef struct { + struct timespec start; + struct timespec enter_wait; + struct timespec end; +} run_timing_t; + +typedef struct { + struct timespec start; + struct timespec start_alloc; + struct timespec end_alloc; + struct timespec start_cpu2gpu; + struct timespec enter_wait_cpu2gpu; + struct timespec end_cpu2gpu; + struct timespec start_gpu2cpu; + struct timespec enter_wait_gpu2cpu; + struct timespec end_gpu2cpu; + run_timing_t *runs; +} device_timing_t; + +double diff_timespec(const struct timespec *time1, const struct timespec *time0) { + return (time1->tv_sec - time0->tv_sec) + + (time1->tv_nsec - time0->tv_nsec) / 1000000000.0; +} + +static void usage(const char *name) { + fprintf(stderr, "USAGE: %s [] [Problem size] [Number of iterations]\n", name); + exit(EXIT_FAILURE); +} + int main(int argc, char *argv[]) { ze_result_t ze_rc; @@ -303,12 +323,29 @@ int main(int argc, char *argv[]) ze_driver_handle_t *allDrivers; int max_devices = 1024*1024, nb_devices = 0; void **device_workspaceA, **device_workspaceC; + int errors = 0; + int N = 4096; + int NRUNS = 3; + device_timing_t *timings; if(argc > 1) { max_devices = atoi(argv[1]); if(max_devices <= 0) { - fprintf(stderr, "USAGE: %s []\n", argv[0]); - return EXIT_FAILURE; + usage(argv[0]); + } + + if(argc > 2) { + N = atoi(argv[2]); + if(N <= 0) { + usage(argv[0]); + } + + if(argc >3) { + NRUNS = atoi(argv[3]); + if(NRUNS <= 0) { + usage(argv[0]); + } + } } } @@ -327,21 +364,80 @@ int main(int argc, char *argv[]) fprintf(stderr, "%d device found in driver %d... Bailing out\n", nb_devices, driverId); return 1; } else { - fprintf(stderr, "STATUS: there are %d GPU devices in driver %d\n", nb, driverId); + fprintf(stderr, "STATUS: there are %d GPU devices in driver %d\n", nb, driverId); nb_devices += nb; + nb_devices = nb_devices <= max_devices ? nb_devices : max_devices; } } free(allDrivers); fprintf(stderr, "%d devices found and initialized\n", nb_devices); + timings = (device_timing_t*)calloc(sizeof(device_timing_t), nb_devices); + for(int i = 0; i < nb_devices; i++) { + clock_gettime(CLOCK_REALTIME, &timings[i].start); + timings[i].runs = (run_timing_t*)calloc(sizeof(run_timing_t), NRUNS); + } + //Allocate GPU memory for each device device_workspaceA = (void**)malloc(sizeof(void*)*nb_devices); device_workspaceC = (void**)malloc(sizeof(void*)*nb_devices); int did = 0; for(int driverId = 0; driverId < (int)driverCount; driverId++) { for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + clock_gettime(CLOCK_REALTIME, &timings[did].start_alloc); device_workspaceA[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N*2); - device_workspaceC[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N); + device_workspaceC[did] = allocate_workspace(&drivers[driverId].devices[deviceId], sizeof(double)*N*N); + clock_gettime(CLOCK_REALTIME, &timings[did].end_alloc); + did++; + } + } + + //Initialize the GPU memory for each device + double *cpuA = (double*)calloc(sizeof(double), N*N); + for(int i = 0; i < N; i++) { + cpuA[i*N+i] = 1.0; + } + double *cpuC = (double*)calloc(sizeof(double), N*N); + did=0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Initialize A and C on device %d of driver %d\n", deviceId, driverId); + void *copy_in = (void*)cpuA; + void *copy_out = (void*)device_workspaceA[did]; + size_t count = N*N*sizeof(double); + clock_gettime(CLOCK_REALTIME, &timings[did].start_cpu2gpu); + ze_rc = zeCommandListReset(device->streams[0].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[0].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(A->GPU) ", ze_rc, { continue; } ); + copy_in = (void*)cpuC; + copy_out = (void*)device_workspaceC[did]; + count = N*N*sizeof(double); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[0].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(C->GPU) ", ze_rc, { continue; } ); + + ze_rc = zeCommandListClose(device->streams[0].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[0].cq, 1, &device->streams[0].cl, device->streams[0].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + + clock_gettime(CLOCK_REALTIME, &timings[did].enter_wait_cpu2gpu); + do { + ze_rc = zeFenceQueryStatus(device->streams[0].fences[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: copies ended on device %d of driver %d\n", deviceId, driverId); + break; + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(10); + } + } while(1); + clock_gettime(CLOCK_REALTIME, &timings[did].end_cpu2gpu); + fprintf(stderr, "STATUS: CPU->GPU copies on device %d of driver %d completed!\n", deviceId, driverId); + } did++; } } @@ -352,28 +448,35 @@ int main(int argc, char *argv[]) for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { device_t *device = &drivers[driverId].devices[deviceId]; if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { - fprintf(stderr, "STATUS: Ready to submit GEMM[%d] on device %d of driver %d\n", run, deviceId, driverId); - fprintf(stderr, "STATUS: Context of driver %d is %s\n", driverId, zeContextGetStatus(drivers[driverId].context) == ZE_RESULT_SUCCESS ? "Fine" : "Broken"); - dpcpp_kernel_GEMM(device->driver->swp, device->swd, device->streams[2].swq, (double*)device_workspaceA[did], &(((double*)device_workspaceC[did])[N*N]), N); - fprintf(stderr, "STATUS: GEMM[%d] submitted on device %d of driver %d\n", run, deviceId, driverId); - - ze_rc = zeCommandListAppendSignalEvent( device->streams[2].cl, device->streams[2].events[0] ); - assert(ZE_RESULT_SUCCESS == ze_rc); - ze_rc = zeCommandListClose(device->streams[2].cl); + fprintf(stderr, "STATUS: Get C back from device %d of driver %d\n", deviceId, driverId); + void *copy_in = (void*)device_workspaceC[did]; + void *copy_out = (void*)cpuC; + size_t count = N*N*sizeof(double); + + for(int i = 0; i < N*N; i++) cpuC[i] = -1.0; + + clock_gettime(CLOCK_REALTIME, &timings[did].start_gpu2cpu); + ze_rc = zeCommandListReset(device->streams[1].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = (ze_result_t)zeCommandListAppendMemoryCopy(device->streams[1].cl, copy_out, copy_in, count, NULL, 0, NULL); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy(GPU->A) ", ze_rc, { continue; } ); + + ze_rc = zeCommandListClose(device->streams[1].cl); LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); - ze_rc = zeCommandQueueExecuteCommandLists(device->streams[2].cq, 1, &device->streams[2].cl, NULL); + ze_rc = zeFenceReset(device->streams[1].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[1].cq, 1, &device->streams[1].cl, device->streams[1].fences[0]); LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); - ze_rc = zeCommandListReset(device->streams[2].cl); - LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + clock_gettime(CLOCK_REALTIME, &timings[did].enter_wait_gpu2cpu); do { - ze_rc = zeEventQueryStatus(device->streams[2].events[0]); + ze_rc = zeFenceQueryStatus(device->streams[1].fences[0]); if( ZE_RESULT_SUCCESS == ze_rc ) { fprintf(stderr, "STATUS: GEMM ended on device %d of driver %d\n", deviceId, driverId); } else if( ZE_RESULT_NOT_READY != ze_rc ) { - LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeEventQueryStatus ", ze_rc, { continue; } ); + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); } else { - usleep(1000); + usleep(10); } } while(1); } else { From ff5f05aed52852e582382daefa25bb810cc140cb Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Tue, 20 Dec 2022 20:05:32 +0000 Subject: [PATCH 13/15] Use level-zero fences to synchronize command lists and command queues, because command lists (or work) submitted to the command queues by SYCL (typically oneMKL) can complete in parallel with events belonging to other command lists. --- .../mca/device/level_zero/device_level_zero.h | 11 ++- .../level_zero/device_level_zero_component.c | 2 +- .../level_zero/device_level_zero_module.c | 76 ++++++++----------- tests/dsl/ptg/cuda/nvlink.jdf | 2 - tests/dsl/ptg/cuda/nvlink_wrapper.c | 16 ++-- 5 files changed, 45 insertions(+), 62 deletions(-) diff --git a/parsec/mca/device/level_zero/device_level_zero.h b/parsec/mca/device/level_zero/device_level_zero.h index 4505f7fd9..cff550017 100644 --- a/parsec/mca/device/level_zero/device_level_zero.h +++ b/parsec/mca/device/level_zero/device_level_zero.h @@ -57,14 +57,13 @@ PARSEC_OBJ_CLASS_DECLARATION(parsec_device_level_zero_module_t); struct parsec_level_zero_exec_stream_s { parsec_gpu_exec_stream_t super; - /* There is exactly one task per active event (max_events being the uppoer bound). - * Upon event completion the complete_stage function associated with the task is + /* There is exactly one task per active fence (max_fences being the uppoer bound). + * Upon fence completion the complete_stage function associated with the task is * called, and this will decide what is going on next with the task. If the task * remains in the system the function is supposed to update it. */ - ze_event_handle_t *events; + ze_fence_handle_t *fences; ze_command_list_handle_t *command_lists; - ze_event_pool_handle_t ze_event_pool; ze_command_queue_handle_t level_zero_cq; parsec_sycl_wrapper_queue_t *swq; }; @@ -76,8 +75,8 @@ struct parsec_level_zero_exec_stream_s { /** * This version is based on 4 streams: one for transfers from the memory to * the GPU, 2 for kernel executions and one for transfers from the GPU into - * the main memory. The synchronization on each stream is based on GPU events, - * such an event indicate that a specific epoch of the lifetime of a task has + * the main memory. The synchronization on each stream is based on GPU fences, + * such a fence indicates that a specific epoch of the lifetime of a task has * been completed. Each type of stream (in, exec and out) has a pending FIFO, * where tasks ready to jump to the respective step are waiting. */ diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c index 0f1cbc231..d7b1958e4 100644 --- a/parsec/mca/device/level_zero/device_level_zero_component.c +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -362,7 +362,7 @@ static int device_level_zero_component_open(void) * Remove all LEVEL_ZERO devices from the PaRSEC available devices, and turn them off. * At the end of this function all LEVEL_ZERO initialization in the context of PaRSEC * should be undone, and pending tasks either completed or transferred to another - * chore (if available), and all LEVEL_ZERO resources (events, streams and memory) + * chore (if available), and all LEVEL_ZERO resources (fences, streams and memory) * released. */ static int device_level_zero_component_close(void) diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c index 1f06158bf..ecd422f97 100644 --- a/parsec/mca/device/level_zero/device_level_zero_module.c +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -259,20 +259,10 @@ int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t PARSEC_OBJ_CONSTRUCT(exec_stream->fifo_pending, parsec_list_t); exec_stream->tasks = (parsec_gpu_task_t**)malloc(exec_stream->max_events * sizeof(parsec_gpu_task_t*)); - level_zero_stream->events = (ze_event_handle_t*)malloc(exec_stream->max_events * sizeof(ze_event_handle_t)); + level_zero_stream->fences = (ze_fence_handle_t*)malloc(exec_stream->max_events * sizeof(ze_fence_handle_t)); level_zero_stream->command_lists = (ze_command_list_handle_t*)malloc(exec_stream->max_events*sizeof(ze_command_list_handle_t)); - // Create event pool - ze_event_pool_desc_t eventPoolDesc = { - ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, - NULL, - ZE_EVENT_POOL_FLAG_HOST_VISIBLE, // all events in pool are visible to Host - 1 // count - }; - zeEventPoolCreate(level_zero_device->driver->ze_context, &eventPoolDesc, 0, NULL, - &level_zero_stream->ze_event_pool); - - /* and the corresponding events */ + /* create the fences and command lists */ for( k = 0; k < exec_stream->max_events; k++ ) { ze_command_list_desc_t commandListDesc = { ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, @@ -280,20 +270,18 @@ int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t computeQueueGroupOrdinal, 0 // flags }; - ze_event_desc_t eventDesc = { - ZE_STRUCTURE_TYPE_EVENT_DESC, - NULL, - 0, // index - 0, // no additional memory/cache coherency required on signal - ZE_EVENT_SCOPE_FLAG_HOST // ensure memory coherency across device and Host after event completes + ze_fence_desc_t fence_desc = { + .stype = ZE_STRUCTURE_TYPE_FENCE_DESC, + .pNext = NULL, + .flags = 0 }; - level_zero_stream->events[k] = NULL; + level_zero_stream->fences[k] = NULL; exec_stream->tasks[k] = NULL; ze_rc = zeCommandListCreate(level_zero_device->driver->ze_context, level_zero_device->ze_device, &commandListDesc, &level_zero_stream->command_lists[k]); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, {goto release_device;} ); - ze_rc = zeEventCreate(level_zero_stream->ze_event_pool, &eventDesc, &(level_zero_stream->events[k])); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventCreate ", ze_rc, {goto release_device;} ); + ze_rc = zeFenceCreate(level_zero_stream->level_zero_cq, &fence_desc, &(level_zero_stream->fences[k])); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceCreate ", ze_rc, {goto release_device;} ); } if(j == 0) { len = asprintf(&exec_stream->name, "h2d(%d)", j); @@ -403,14 +391,13 @@ int parsec_level_zero_module_init( int dev_id, parsec_device_level_zero_driver_t if( NULL != exec_stream->tasks ) { free(exec_stream->tasks); exec_stream->tasks = NULL; } - if( NULL != level_zero_stream->events ) { + if( NULL != level_zero_stream->fences ) { for( k = 0; k < exec_stream->max_events; k++ ) { - if( NULL != level_zero_stream->events[k] ) { - (void)zeEventDestroy(level_zero_stream->events[k]); + if( NULL != level_zero_stream->fences[k] ) { + (void)zeFenceDestroy(level_zero_stream->fences[k]); } } - free(level_zero_stream->events); level_zero_stream->events = NULL; - zeEventPoolDestroy(level_zero_stream->ze_event_pool); + free(level_zero_stream->fences); level_zero_stream->fences = NULL; } if( NULL != exec_stream->name ) { free(exec_stream->name); exec_stream->name = NULL; @@ -458,18 +445,15 @@ parsec_level_zero_module_fini(parsec_device_module_t* device) for( k = 0; k < exec_stream->max_events; k++ ) { assert( NULL == exec_stream->tasks[k] ); - status = zeEventDestroy(level_zero_stream->events[k]); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "(parsec_level_zero_device_fini) level_zeroEventDestroy ", status, {} ); + status = zeFenceDestroy(level_zero_stream->fences[k]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(parsec_level_zero_device_fini) zeFenceDestroy ", status, {} ); status = zeCommandListDestroy(level_zero_stream->command_lists[k]); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListDestroy ", status, {} ); } - zeEventPoolDestroy(level_zero_stream->ze_event_pool); exec_stream->max_events = 0; - free(level_zero_stream->events); level_zero_stream->events = NULL; + free(level_zero_stream->fences); level_zero_stream->fences = NULL; free(exec_stream->tasks); exec_stream->tasks = NULL; free(exec_stream->fifo_pending); exec_stream->fifo_pending = NULL; - /* Release the stream */ - ze_result_t ze_rc; /* Deleting the sycl queue wrapper and/or the command queue conflicts with the * cleaning procedure of the Level Zero runtime... Didn't find a way to do it * cleanly. Don't cleanup for now... */ @@ -550,9 +534,6 @@ parsec_level_zero_memory_reserve( parsec_device_level_zero_module_t* level_zero_ } initial_free_mem = devProperties.maxMemAllocSize < devMemProperties[memIndex].totalSize ? devProperties.maxMemAllocSize : devMemProperties[memIndex].totalSize; - fprintf(stderr, "level-zero device %s: initial_free_mem is %lu (device max alloc size is %lu, device memory " - "property total size for memory bank %d is %lu)", gpu_device->super.name, initial_free_mem, - devProperties.maxMemAllocSize, memIndex, devMemProperties[memIndex].totalSize); free(devMemProperties); devMemProperties = NULL; if( number_blocks != -1 ) { @@ -1684,7 +1665,7 @@ parsec_gpu_callback_complete_push(parsec_device_gpu_module_t *gpu_device, #endif const parsec_flow_t *flow; /** - * Even though level_zero event return success, the PUSH may not be + * Even though level_zero fence returns success, the PUSH may not be * completed if no PUSH is required by this task and the PUSH is * actually done by another task, so we need to check if the data is * actually ready to use @@ -1886,7 +1867,7 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, progress_fct = upstream_progress_fct; if( NULL != stream->tasks[stream->end] ) { - rc = zeEventQueryStatus(level_zero_stream->events[stream->end]); + rc = zeFenceQueryStatus(level_zero_stream->fences[stream->end]); if( ZE_RESULT_SUCCESS == rc ) { /* Save the task for the next step */ task = *out_task = stream->tasks[stream->end]; @@ -1915,7 +1896,7 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, return rc; } if( ZE_RESULT_NOT_READY != rc ) { - PARSEC_LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeEventQueryStatus ", rc, + PARSEC_LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", rc, {return PARSEC_HOOK_RETURN_AGAIN;} ); } } @@ -1951,6 +1932,14 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, } #endif /* defined(PARSEC_DEBUG_PARANOID) */ } + + /* Prepare the command list and the Fence before progressing the function since it may + * enqueue stuff to the command list */ + ze_result_t ze_rc = zeCommandListReset(level_zero_stream->command_lists[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { } ); + ze_rc = zeFenceReset(level_zero_stream->fences[stream->start]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { } ); + rc = progress_fct( gpu_device, task, stream ); if( 0 > rc ) { if( PARSEC_HOOK_RETURN_AGAIN != rc && @@ -1973,21 +1962,16 @@ progress_stream( parsec_device_gpu_module_t* gpu_device, return PARSEC_HOOK_RETURN_DONE; } /** - * Do not skip the level_zero event generation. The problem is that some of the inputs + * Do not skip the level_zero fence generation. The problem is that some of the inputs * might be in the pipe of being transferred to the GPU. If we activate this task * too early, it might get executed before the data is available on the GPU. * Obviously, this lead to incorrect results. */ - rc = zeCommandListAppendSignalEvent( level_zero_stream->command_lists[stream->start], level_zero_stream->events[stream->start] ); - assert(ZE_RESULT_SUCCESS == rc); - ze_result_t ze_rc = zeCommandListClose(level_zero_stream->command_lists[stream->start]); + ze_rc = zeCommandListClose(level_zero_stream->command_lists[stream->start]); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { } ); - ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->command_lists[stream->start], NULL); + ze_rc = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->command_lists[stream->start], level_zero_stream->fences[stream->start]); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { } ); - ze_rc = zeCommandQueueSynchronize( level_zero_stream->level_zero_cq, 5000000 ); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueSynchronize ", ze_rc, { } ); - stream->tasks[stream->start] = task; stream->start = (stream->start + 1) % stream->max_events; PARSEC_DEBUG_VERBOSE(20, parsec_gpu_output_stream, diff --git a/tests/dsl/ptg/cuda/nvlink.jdf b/tests/dsl/ptg/cuda/nvlink.jdf index f67b8a22a..5b473e20b 100644 --- a/tests/dsl/ptg/cuda/nvlink.jdf +++ b/tests/dsl/ptg/cuda/nvlink.jdf @@ -183,7 +183,6 @@ BODY [type=DPCPP } catch (...) { parsec_warning("OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); } - fprintf(stderr, "GEMM1(%d, %d, %d) has been scheduled on OneAPI MKL BLAS using the DPC++ driver\n", m, g, r); } END @@ -258,7 +257,6 @@ BODY [type=DPCPP } catch (...) { parsec_warning("OneAPI MKL BLAS GEMM2 throws unexpected exception that is also badly formatted..."); } - fprintf(stderr, "GEMM2(%d, %d, %d) has been scheduled on OneAPI MKL BLAS using the DPC++ driver\n", m, g, r); } END diff --git a/tests/dsl/ptg/cuda/nvlink_wrapper.c b/tests/dsl/ptg/cuda/nvlink_wrapper.c index 7ee2bb9c3..5b10b71e9 100644 --- a/tests/dsl/ptg/cuda/nvlink_wrapper.c +++ b/tests/dsl/ptg/cuda/nvlink_wrapper.c @@ -248,23 +248,25 @@ parsec_taskpool_t* testing_nvlink_New( parsec_context_t *ctx, int depth, int mb PARSEC_CUDA_CHECK_ERROR( "(nvlink_wrapper) cudaMemcpy ", status, {return NULL;} ); #elif defined(PARSEC_HAVE_LEVEL_ZERO) parsec_level_zero_exec_stream_t* level_zero_stream = (parsec_level_zero_exec_stream_t*)level_zero_device->super.exec_stream[0]; - ze_event_handle_t copySignalEvent = level_zero_stream->events[0]; - status = zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[0], gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, copySignalEvent, 0, NULL); + ze_fence_handle_t copySignalFence = level_zero_stream->fences[0]; + status = zeFenceReset(copySignalFence); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", status, { return NULL; } ); + status = zeCommandListReset(level_zero_stream->command_lists[0]); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", status, { return NULL; } ); + status = zeCommandListAppendMemoryCopy(level_zero_stream->command_lists[0], gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts, NULL, 0, NULL); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListAppendMemoryCopy ", status, { return NULL; } ); status = zeCommandListClose(level_zero_stream->command_lists[0]); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", status, { return NULL; } ); - status = zeCommandQueueExecuteCommandLists(level_zero_stream->command_lists[0], 1, &level_zero_stream->command_lists[0], NULL); + status = zeCommandQueueExecuteCommandLists(level_zero_stream->level_zero_cq, 1, &level_zero_stream->command_lists[0], copySignalFence); PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", status, { return NULL; } ); while(1) { - status = zeEventQueryStatus(copySignalEvent); + status = zeFenceQueryStatus(copySignalFence); if(status == ZE_RESULT_SUCCESS) break; if(status != ZE_RESULT_NOT_READY) - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeEventHostSynchronize ", status, { break; } ); + PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeFenceQueryStatus ", status, { break; } ); usleep(1000); } - status = zeCommandListReset(level_zero_stream->command_lists[0]); - PARSEC_LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", status, { return NULL; } ); #else memcpy(gpu_copy->device_private, cpu_copy->device_private, dta->nb_elts); #endif From d4594d3119a6fc0db5fde2c102a93c93755f49ee Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Tue, 17 Jan 2023 19:40:23 +0000 Subject: [PATCH 14/15] Define the set of globals in DPC++ code after the includse happen to avoid polluting their namespace; cleanup some unused variables --- parsec/interfaces/ptg/ptg-compiler/jdf2c.c | 81 ++++++++++------------ 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c index 338a6e439..32d070c57 100644 --- a/parsec/interfaces/ptg/ptg-compiler/jdf2c.c +++ b/parsec/interfaces/ptg/ptg-compiler/jdf2c.c @@ -1561,15 +1561,11 @@ static void jdf_minimal_code_before_prologue(const jdf_t *jdf) static void jdf_dump_internal_structure(string_arena_t *sa, jdf_t *jdf) { int nbfunctions = 0, need_profile = 0; - string_arena_t *sa1, *sa2; jdf_function_entry_t* f; jdf_param_list_t *pl; JDF_COUNT_LIST_ENTRIES(jdf->functions, jdf_function_entry_t, next, nbfunctions); - sa1 = string_arena_new(64); - sa2 = string_arena_new(64); - string_arena_add_string(sa, "#include \"%s.h\"\n\n" "struct __parsec_%s_internal_taskpool_s {\n" " parsec_%s_taskpool_t super;\n" @@ -1604,6 +1600,24 @@ static void jdf_dump_internal_structure(string_arena_t *sa, jdf_t *jdf) string_arena_add_string(sa, "};\n\n"); } +static int jdf_has_dpcpp_chore(const jdf_t *jdf, const char *fname) +{ + jdf_function_entry_t *f; + jdf_body_t* body; + jdf_def_list_t *type_property; + + for(f = jdf->functions; f != NULL; f = f->next) { + if( (NULL != fname) && strcmp(f->fname, fname) ) continue; + for(body = f->bodies; body != NULL; body = body->next) { + jdf_find_property(body->properties, "type", &type_property); + if( NULL != type_property && !strcmp(type_property->expr->jdf_var, "DPCPP")) + return 1; + } + if(NULL != fname) return 0; + } + return 0; +} + static void jdf_generate_structure(jdf_t *jdf) { int nbfunctions, need_profile = 0; @@ -1639,6 +1653,9 @@ static void jdf_generate_structure(jdf_t *jdf) if( 1 < strlen(string_arena_get_string(sa1)) ) { coutput("/* Globals */\n%s\n", string_arena_get_string(sa1)); } + if( 1 < strlen(string_arena_get_string(sa1)) && jdf_has_dpcpp_chore(jdf, NULL) ) { + dpcpp_output("/* Globals */\n%s\n", string_arena_get_string(sa1)); + } coutput("static inline int parsec_imin(int a, int b) { return (a <= b) ? a : b; };\n\n" "static inline int parsec_imax(int a, int b) { return (a >= b) ? a : b; };\n\n"); @@ -4861,7 +4878,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e string_arena_t *sa_range_multiplier = string_arena_new(64); jdf_variable_list_t *vl; expr_info_t info = EMPTY_EXPR_INFO; - int idx; if( !(f->user_defines & JDF_FUNCTION_HAS_UD_MAKE_KEY) ) { coutput("static inline parsec_key_t %s(const parsec_taskpool_t *tp, const parsec_assignment_t *as)\n" @@ -4884,7 +4900,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e info.sa = sa_range_multiplier; info.assignments = "assignment"; - idx = 0; for(vl = f->locals; vl != NULL; vl = vl->next) { string_arena_init(sa_range_multiplier); @@ -4916,7 +4931,6 @@ static void jdf_generate_hashfunction_for(const jdf_t *jdf, const jdf_function_e */ coutput(" (void)%s;\n", vl->name); } - idx++; } string_arena_init(sa_range_multiplier); @@ -6613,25 +6627,6 @@ static int jdf_has_cuda_chore(const jdf_t *jdf, const char *fname) return 0; } -static int jdf_has_dpcpp_chore(const jdf_t *jdf, const char *fname) -{ - jdf_function_entry_t *f; - jdf_body_t* body; - jdf_def_list_t *type_property; - - for(f = jdf->functions; f != NULL; f = f->next) { - if( (NULL != fname) && strcmp(f->fname, fname) ) continue; - for(body = f->bodies; body != NULL; body = body->next) { - jdf_find_property(body->properties, "type", &type_property); - if( NULL != type_property && !strcmp(type_property->expr->jdf_var, "DPCPP")) - return 1; - } - if(NULL != fname) return 0; - } - return 0; -} - - static void jdf_generate_code_hook_cuda(const jdf_t *jdf, const jdf_function_entry_t *f, const jdf_body_t* body, @@ -8054,7 +8049,7 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, string_arena_t *sa_tmp_type_r = string_arena_new(256); string_arena_t *sa_temp_r = string_arena_new(1024); - int depnb, last_datatype_idx; + int last_datatype_idx; assignment_info_t ai; expr_info_t info = EMPTY_EXPR_INFO; int nb_open_ldef; @@ -8104,7 +8099,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, for(fl = f->dataflow; fl != NULL; fl = fl->next) { flowempty = 1; flowtomem = 0; - depnb = 0; last_datatype_idx = -1; string_arena_init(sa_coutput); string_arena_init(sa_deps); @@ -8338,8 +8332,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, jdf_dump_context_assignment(sa1, jdf, f, fl, string_arena_get_string(sa_ontask), dl->guard->calltrue, dl, JDF_OBJECT_LINENO(dl), " ", "nc")); - depnb++; - string_arena_init(sa_ontask); string_arena_add_string(sa_ontask, "if( PARSEC_ITERATE_STOP == ontask(es, &nc, (const parsec_task_t *)this_task, &%s, &data, rank_src, rank_dst, vpid_dst," @@ -8360,7 +8352,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, "\n"); } } else { - depnb++; string_arena_init(sa_ontask); string_arena_add_string(sa_ontask, "if( PARSEC_ITERATE_STOP == ontask(es, &nc, (const parsec_task_t *)this_task, &%s, &data, rank_src, rank_dst, vpid_dst," @@ -8391,7 +8382,6 @@ jdf_generate_code_iterate_successors_or_predecessors(const jdf_t *jdf, } break; } - depnb++; /* Dump the previous dependencies */ OUTPUT_PREV_DEPS((1U << dl->dep_index), sa_datatype, sa_deps); @@ -8793,8 +8783,7 @@ int jdf2c(const char *output_c, const char *output_h, dpcppfile = NULL; if( jdf_has_dpcpp_chore(jdf, NULL) ) { - string_arena_t *sa1, *sa2; - sa1 = string_arena_new(64); + string_arena_t *sa2; sa2 = string_arena_new(64); dpcppfile = fopen(jdf_dpcppfilename, "w"); if( dpcppfile == NULL ) { @@ -8803,9 +8792,6 @@ int jdf2c(const char *output_c, const char *output_h, goto err; } - UTIL_DUMP_LIST(sa1, jdf->globals, next, - dump_globals, sa2, "", "#define ", "\n", "\n"); - dpcpp_output("#include \"parsec.h\"\n" "#include \"level_zero/ze_api.h\"\n" "#include \"sycl/ext/oneapi/backend/level_zero.hpp\"\n" @@ -8815,10 +8801,7 @@ int jdf2c(const char *output_c, const char *output_h, "#include \"parsec/mca/device/device_gpu.h\"\n" "#include \"parsec/mca/device/level_zero/device_level_zero.h\"\n" "#include \"parsec/mca/device/level_zero/device_level_zero_dpcpp.h\"\n" - "\n" - "%s\n", - string_arena_get_string(sa1)); - string_arena_free(sa1); + "\n"); string_arena_free(sa2); } @@ -8953,12 +8936,10 @@ int jdf2c(const char *output_c, const char *output_h, /* Dump references to arenas_datatypes array */ struct jdf_name_list* g; - int datatype_index = 0; for( g = jdf->datatypes; NULL != g; g = g->next ) { coutput("#define PARSEC_%s_%s_ADT (&__parsec_tp->super.arenas_datatypes[PARSEC_%s_%s_ADT_IDX])\n", jdf_basename, g->name, jdf_basename, g->name); - datatype_index++; } jdf_generate_structure(jdf); @@ -8985,6 +8966,20 @@ int jdf2c(const char *output_c, const char *output_h, free_name_placeholders(); + /* Sometimes (typically because of a try {} catch block), sycl re-includes some definitions at the end of the file + * these definitions can be polluted by macros. Undefine the macros. */ + if( jdf_has_dpcpp_chore(jdf, NULL) ) { + string_arena_t *sa1; + sa1 = string_arena_new(64); + + UTIL_DUMP_LIST_FIELD(sa1, jdf->globals, next, name, + dump_string, NULL, + "", "#undef ", "\n", "\n"); + + dpcpp_output("%s\n", string_arena_get_string(sa1)); + string_arena_free(sa1); + } + /** * Dump all the epilogue sections */ From 06291c2156a4738b89396c48aadd361abd50947c Mon Sep 17 00:00:00 2001 From: Thomas Herault Date: Tue, 17 Jan 2023 19:41:42 +0000 Subject: [PATCH 15/15] Install LevelZero driver files; setup the environment to find the same LevelZero library as at compile time in PaRSECConfig.cmake --- CMakeLists.txt | 2 + cmake_modules/PaRSECConfig.cmake.in | 17 +++ parsec/mca/device/CMakeLists.txt | 4 +- .../level_zero/device_level_zero_component.c | 4 +- .../level_zero/device_level_zero_module.c | 3 +- parsec/mca/device/transfer_gpu.c | 8 +- tests/subsystem/level_zero/interface.dpcpp.C | 46 +++++- tests/subsystem/level_zero/interface.dpcpp.h | 8 +- tests/subsystem/level_zero/kernel.dpcpp.C | 31 ++-- tests/subsystem/level_zero/main.c | 141 ++++++++++++++---- 10 files changed, 206 insertions(+), 58 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a7047cfab..74e551315 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1232,6 +1232,8 @@ install(FILES ${PROJECT_SOURCE_DIR}/cmake_modules/FindPAPI.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/FindOTF2.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/FindHWLOC.cmake + ${PROJECT_SOURCE_DIR}/cmake_modules/Findlevel-zero.cmake + ${PROJECT_SOURCE_DIR}/cmake_modules/FindDPCPP.cmake ${PROJECT_SOURCE_DIR}/cmake_modules/ParsecCompilePTG.cmake DESTINATION ${PARSEC_INSTALL_CMAKEDIR}) diff --git a/cmake_modules/PaRSECConfig.cmake.in b/cmake_modules/PaRSECConfig.cmake.in index 429ee4a33..b37e855f1 100644 --- a/cmake_modules/PaRSECConfig.cmake.in +++ b/cmake_modules/PaRSECConfig.cmake.in @@ -67,6 +67,23 @@ if(@PARSEC_HAVE_CUDA@) SET(PARSEC_HAVE_CUDA TRUE) endif(@PARSEC_HAVE_CUDA@) +if(@PARSEC_HAVE_LEVEL_ZERO@) + if(NOT "@LEVEL_ZERO_ROOT_DIR@" EQUAL "") + set(LEVEL_ZERO_ROOT_DIR "@LEVEL_ZERO_ROOT_DIR@") + endif(NOT "@LEVEL_ZERO_ROOT_DIR@" EQUAL "") + find_package(level-zero) + find_package(DPCPP) + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero") + if (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/") + set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero") + message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}") + message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}") + else(LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) + message(STATUS "Could not find both level-zero and DPCPP: level-zero says ${LEVEL_ZERO_FOUND}, DPCPP says ${DPCPP_EXECUTABLE}") + endif (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP) +endif(@PARSEC_HAVE_LEVEL_ZERO@) + if(@PARSEC_PROF_TRACE@) # Nothing exportable here, if this test succeed then PaRSEC supports tracing endif(@PARSEC_PROF_TRACE@) diff --git a/parsec/mca/device/CMakeLists.txt b/parsec/mca/device/CMakeLists.txt index 6f7b7ce77..30c71e0f2 100644 --- a/parsec/mca/device/CMakeLists.txt +++ b/parsec/mca/device/CMakeLists.txt @@ -1,8 +1,8 @@ set(MCA_${COMPONENT}_SOURCES mca/device/device.c) -if( PARSEC_HAVE_CUDA ) +if( PARSEC_HAVE_CUDA OR PARSEC_HAVE_LEVEL_ZERO ) LIST(APPEND MCA_${COMPONENT}_SOURCES mca/device/device_gpu.c mca/device/transfer_gpu.c) -endif( PARSEC_HAVE_CUDA ) +endif( PARSEC_HAVE_CUDA OR PARSEC_HAVE_LEVEL_ZERO ) set_property(TARGET parsec APPEND PROPERTY diff --git a/parsec/mca/device/level_zero/device_level_zero_component.c b/parsec/mca/device/level_zero/device_level_zero_component.c index d7b1958e4..51ae04d57 100644 --- a/parsec/mca/device/level_zero/device_level_zero_component.c +++ b/parsec/mca/device/level_zero/device_level_zero_component.c @@ -32,8 +32,8 @@ static int device_level_zero_component_close(void); static int device_level_zero_component_query(mca_base_module_2_0_0_t **module, int *priority); static int device_level_zero_component_register(void); -int use_level_zero_index, use_level_zero; -int parsec_cuda_sort_pending = 0, parsec_cuda_max_streams = PARSEC_GPU_MAX_STREAMS; +int parsec_device_level_zero_index, parsec_device_level_zero_enabled; +int parsec_level_zero_sort_pending = 0, parsec_level_zero_max_streams = PARSEC_GPU_MAX_STREAMS; int level_zero_mask, level_zero_nvlink_mask; int level_zero_memory_block_size, level_zero_memory_percentage, level_zero_memory_number_of_blocks; diff --git a/parsec/mca/device/level_zero/device_level_zero_module.c b/parsec/mca/device/level_zero/device_level_zero_module.c index ecd422f97..7ffd94304 100644 --- a/parsec/mca/device/level_zero/device_level_zero_module.c +++ b/parsec/mca/device/level_zero/device_level_zero_module.c @@ -1599,7 +1599,6 @@ static parsec_task_class_t parsec_level_zero_d2d_complete_tc = { .initial_data = NULL, .final_data = NULL, .data_affinity = NULL, - .key_generator = NULL, .key_functions = NULL, .make_key = NULL, .get_datatype = NULL, @@ -2612,7 +2611,7 @@ parsec_level_zero_kernel_scheduler( parsec_execution_stream_t *es, /* Everything went fine so far, the result is correct and back in the main memory */ PARSEC_LIST_ITEM_SINGLETON(gpu_task); if (gpu_task->task_type == GPU_TASK_TYPE_D2HTRANSFER) { - parsec_gpu_w2r_task_fini(gpu_device, gpu_task, es); + parsec_gpu_complete_w2r_task(gpu_device, gpu_task, es); gpu_task = progress_task; goto fetch_task_from_shared_queue; } diff --git a/parsec/mca/device/transfer_gpu.c b/parsec/mca/device/transfer_gpu.c index 439a84724..68fb7aa61 100644 --- a/parsec/mca/device/transfer_gpu.c +++ b/parsec/mca/device/transfer_gpu.c @@ -18,9 +18,9 @@ #include "parsec/utils/output.h" #include "parsec/scheduling.h" -#if !defined(PARSEC_HAVE_CUDA) -#error This file should not be included in a non-CUDA build -#endif /* !defined(PARSEC_HAVE_CUDA) */ +#if !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_LEVEL_ZERO) +#error This file should not be included if no GPU device is supported +#endif /* !defined(PARSEC_HAVE_CUDA) && !defined(PARSEC_HAVE_LEVEL_ZERO) */ /** * Entirely local tasks that should only be used to move data between a device and the main memory. Such @@ -136,7 +136,7 @@ static const __parsec_chore_t __gpu_d2h_task_chores[] = { #if defined(PARSEC_HAVE_LEVEL_ZERO) {.type = PARSEC_DEV_LEVEL_ZERO, .evaluate = NULL, - .hook = (parsec_hook_t *) hook_of_GPU_d2h_task}, + .hook = (parsec_hook_t *) hook_of_gpu_d2h_task}, #endif {.type = PARSEC_DEV_NONE, .evaluate = NULL, diff --git a/tests/subsystem/level_zero/interface.dpcpp.C b/tests/subsystem/level_zero/interface.dpcpp.C index 46f802c69..8abe7c5e8 100644 --- a/tests/subsystem/level_zero/interface.dpcpp.C +++ b/tests/subsystem/level_zero/interface.dpcpp.C @@ -1,10 +1,14 @@ #include "level_zero/ze_api.h" #include "interface.dpcpp.h" -sycl_wrapper_t *sycl_queue_create(ze_driver_handle_t ze_driver, - ze_device_handle_t ze_device, - ze_context_handle_t ze_context, - ze_command_queue_handle_t ze_queue) +sycl_wrapper_driver_t *sycl_wrapper_platform_create(ze_driver_handle_t ze_driver) +{ + sycl_wrapper_driver_t *res = new sycl_wrapper_driver_t; + res->platform = sycl::make_platform(ze_driver); + return res; +} + +void sycl_wrapper_platform_add_context(sycl_wrapper_driver_t *swp, ze_context_handle_t ze_context, sycl_wrapper_device_t **swd, uint32_t num_device) { std::vectordevices; @@ -12,15 +16,41 @@ sycl_wrapper_t *sycl_queue_create(ze_driver_handle_t ze_driver, devices.push_back(swd[i]->device); } sycl::backend_input_t hContextInteropInput = {ze_context, devices}; - res->context = sycl::make_context(hContextInteropInput); - res->queue = sycl::make_queue(ze_queue, res->context); + swp->context = sycl::make_context(hContextInteropInput); +} + +sycl_wrapper_device_t *sycl_wrapper_device_create(ze_device_handle_t ze_device) +{ + sycl_wrapper_device_t *res = new sycl_wrapper_device_t; + + res->device = sycl::make_device(ze_device); return res; } -int sycl_queue_destroy(sycl_wrapper_t *sycl_obj) +sycl_wrapper_queue_t *sycl_wrapper_queue_create(sycl_wrapper_driver_t *swp, sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue) +{ + sycl_wrapper_queue_t *swq = new sycl_wrapper_queue_t; + sycl::backend_input_t hQueueInteropInput = { ze_queue, swd->device }; + swq->queue = sycl::make_queue(hQueueInteropInput, swp->context); + return swq; +} + +int sycl_wrapper_driver_destroy(sycl_wrapper_driver_t *swp) +{ + delete swp; + return 0; +} + +int sycl_wrapper_device_destroy(sycl_wrapper_device_t *swd) +{ + delete swd; + return 0; +} + +int sycl_wrapper_queue_destroy(sycl_wrapper_queue_t *swq) { - delete sycl_obj; + delete swq; return 0; } diff --git a/tests/subsystem/level_zero/interface.dpcpp.h b/tests/subsystem/level_zero/interface.dpcpp.h index 79dc34f62..04aa2f380 100644 --- a/tests/subsystem/level_zero/interface.dpcpp.h +++ b/tests/subsystem/level_zero/interface.dpcpp.h @@ -29,11 +29,9 @@ void sycl_wrapper_platform_add_context(sycl_wrapper_driver_t *swp, ze_context_ha sycl_wrapper_device_t *sycl_wrapper_device_create(ze_device_handle_t ze_device); sycl_wrapper_queue_t *sycl_wrapper_queue_create(sycl_wrapper_driver_t *swp, sycl_wrapper_device_t *swd, ze_command_queue_handle_t ze_queue); -sycl_wrapper_t *sycl_queue_create(ze_driver_handle_t ze_driver, - ze_device_handle_t ze_device, - ze_context_handle_t ze_context, - ze_command_queue_handle_t ze_queue); -int sycl_queue_destroy(sycl_wrapper_t *_queue); +int sycl_wrapper_driver_destroy(sycl_wrapper_driver_t *swp); +int sycl_wrapper_device_destroy(sycl_wrapper_device_t *swd); +int sycl_wrapper_queue_destroy(sycl_wrapper_queue_t *swq); #if defined(c_plusplus) || defined(__cplusplus) } diff --git a/tests/subsystem/level_zero/kernel.dpcpp.C b/tests/subsystem/level_zero/kernel.dpcpp.C index eea5c23bb..0ab12e76a 100644 --- a/tests/subsystem/level_zero/kernel.dpcpp.C +++ b/tests/subsystem/level_zero/kernel.dpcpp.C @@ -11,14 +11,14 @@ extern "C" { int mb); } -int dpcpp_kernel_GEMM(void *_sw, +int dpcpp_kernel_GEMM(sycl_wrapper_driver_t *swp, + sycl_wrapper_device_t *swd, + sycl_wrapper_queue_t *swq, const double *A, double *C, int mb) { - sycl_wrapper_t *sw = reinterpret_cast(_sw); - - double alpha=0.0; + double alpha=1.0; double beta=1.0; /* sycl::backend_input_t> hBufferInteropInputA = { (void*)A, sycl::ext::oneapi::level_zero::ownership::keep }; sycl::buffer bA = sycl::make_buffer(hBufferInteropInputA, swp->context); @@ -33,17 +33,17 @@ int dpcpp_kernel_GEMM(void *_sw, A, mb, beta, C, mb); } catch (const oneapi::mkl::invalid_argument &e) { - fprintf(stderr, "OneAPI MKL BLAS GEMM throws invalid argument exception"); + fprintf(stderr, "OneAPI MKL BLAS GEMM throws invalid argument exception\n"); } catch (const oneapi::mkl::unsupported_device &e) { - fprintf(stderr, "OneAPI MKL BLAS GEMM throws unsuported device exception"); + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unsuported device exception\n"); } catch (const oneapi::mkl::host_bad_alloc &e) { - fprintf(stderr, "OneAPI MKL BLAS GEMM throws host bad allocation exception"); + fprintf(stderr, "OneAPI MKL BLAS GEMM throws host bad allocation exception\n"); } catch (const oneapi::mkl::device_bad_alloc &e) { - fprintf(stderr, "OneAPI MKL BLAS GEMM throws device bad allocation exception"); + fprintf(stderr, "OneAPI MKL BLAS GEMM throws device bad allocation exception\n"); } catch (const oneapi::mkl::unimplemented &e) { - fprintf(stderr, "OneAPI MKL BLAS GEMM throws unimplemented exception"); + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unimplemented exception\n"); } catch (const std::exception& e) { - fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception"); + fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception '%s'\n", e.what()); } catch (...) { fprintf(stderr, "OneAPI MKL BLAS GEMM throws unexpected exception that is also badly formatted..."); } @@ -51,3 +51,14 @@ int dpcpp_kernel_GEMM(void *_sw, return 0; } +/* +void *sycl_malloc(sycl_wrapper_t *sw, size_t size) +{ + return sycl::malloc_device(size, sw->queue); +} + +void sycl_free(sycl_wrapper_t *sw, void *ptr) +{ + sycl::free(ptr, sw->queue); +} +*/ diff --git a/tests/subsystem/level_zero/main.c b/tests/subsystem/level_zero/main.c index 93e0e5844..d6cce9635 100644 --- a/tests/subsystem/level_zero/main.c +++ b/tests/subsystem/level_zero/main.c @@ -1,11 +1,8 @@ #include #include -<<<<<<< HEAD -======= #include #include #include ->>>>>>> f5f749053... Fix the subsystem test. Need to backport fixes in the MCA device #include #include "interface.dpcpp.h" @@ -26,7 +23,7 @@ struct stream_s; typedef struct stream_s { int immediate; - void *sq; + sycl_wrapper_queue_t *swq; ze_command_queue_handle_t cq; ze_command_list_handle_t cl; struct device_s *device; @@ -79,11 +76,15 @@ static int init_device(device_t *device, ze_device_handle_t gpuDevice) for( uint32_t i = 0; i < cmdqueueGroupCount && (computeQueueGroupOrdinal == cmdqueueGroupCount || copyQueueGroupOrdinal == cmdqueueGroupCount); ++i ) { - if( computeQueueGroupOrdinal == cmdqueueGroupCount && cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE ) { - computeQueueGroupOrdinal = i; + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE ) { + fprintf(stderr, "INFO: cmdqueueGroup number %d can be used as compute queue\n", i); + if(cmdqueueGroupCount == computeQueueGroupOrdinal) + computeQueueGroupOrdinal = i; } - if( copyQueueGroupOrdinal == cmdqueueGroupCount && cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY ) { - copyQueueGroupOrdinal = i; + if( cmdqueueGroupProperties[ i ].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY ) { + fprintf(stderr, "INFO: cmdqueueGroup number %d can be used as copy queue\n", i); + if(cmdqueueGroupCount == copyQueueGroupOrdinal) + copyQueueGroupOrdinal = i; } } if( computeQueueGroupOrdinal == cmdqueueGroupCount ) { @@ -94,6 +95,7 @@ static int init_device(device_t *device, ze_device_handle_t gpuDevice) fprintf(stderr, "level zero device: unable to find a Queue Group with COMPUTE flag"); return -1; } + fprintf(stderr, "INFO: computeQueueGroupOrdinal = %d, copyQueueGroupOrdinal = %d\n", computeQueueGroupOrdinal, copyQueueGroupOrdinal); for(int j = 0; j < NB_STREAMS; j++ ) { ze_command_queue_desc_t commandQueueDesc = { @@ -106,7 +108,7 @@ static int init_device(device_t *device, ze_device_handle_t gpuDevice) ZE_COMMAND_QUEUE_PRIORITY_NORMAL }; device->streams[j].device = device; - if( j < 2 ) { + if( 0 && j < 2 ) { device->streams[j].immediate = 1; commandQueueDesc.ordinal = copyQueueGroupOrdinal; ze_rc = zeCommandListCreateImmediate(device->driver->context, gpuDevice, @@ -120,17 +122,11 @@ static int init_device(device_t *device, ze_device_handle_t gpuDevice) &commandQueueDesc, &device->streams[j].cq); LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueCreate ", ze_rc, { return -1;} ); ze_command_list_desc_t commandListDesc = { - ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC, - NULL, - computeQueueGroupOrdinal, 0 // flags }; ze_rc = zeCommandListCreate(device->driver->context, gpuDevice, &commandListDesc, &device->streams[j].cl); LEVEL_ZERO_CHECK_ERROR( "zeCommandListCreate ", ze_rc, { return -1;} ); - device->streams[j].sq = sycl_queue_create(device->driver->driver, gpuDevice, device->driver->context, device->streams[j].cq); - if(NULL == device->streams[j].sq) - return -1; } for(int k = 0; k < MAX_FENCES; k++ ) { @@ -157,10 +153,6 @@ static int init_driver(driver_t *driver, int maxDevices) ze_rc = zeDeviceGet(driver->driver, &deviceCount, NULL); LEVEL_ZERO_CHECK_ERROR( "zeDeviceGet (count) ", ze_rc, { return -1; } ); - - driver->nb_devices = 0; - driver->devices = NULL; - if(deviceCount == 0) return 0; @@ -262,7 +254,6 @@ static void *allocate_workspace(device_t *device, size_t size) if( memIndex == -1 || devMemProperties[memIndex].totalSize < devMemProperties[i].totalSize) memIndex = i; } - free(devMemProperties); devMemProperties = NULL; if( size > devMemProperties[memIndex].totalSize ) { /** Handle the case of jokers who require more than 100% of memory, @@ -273,6 +264,10 @@ static void *allocate_workspace(device_t *device, size_t size) size, devMemProperties[memIndex].totalSize); return NULL; } + free(devMemProperties); devMemProperties = NULL; + + /*device_ptr = sycl_malloc(device->streams[2].sw, size); + assert(NULL != device_ptr); */ ze_device_mem_alloc_desc_t memAllocDesc = { .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, .pNext = NULL, @@ -349,6 +344,9 @@ int main(int argc, char *argv[]) } } + ze_rc = zeInit( 0 ); + LEVEL_ZERO_CHECK_ERROR( "zeInit ", ze_rc, { return -1; }); + // Discover all the driver instances ze_rc = zeDriverGet(&driverCount, NULL); LEVEL_ZERO_CHECK_ERROR( "zeDriverGet (count) ", ze_rc, { return 1; } ); @@ -442,7 +440,49 @@ int main(int argc, char *argv[]) } } - //Do a GEMM (blocking) on each device, and wait for its completion -- yes, memory is not initialized. + for(int run = 0; run < NRUNS; run++) { + //Do a GEMM (blocking) on each device, and wait for its completion -- yes, memory is not initialized. + did = 0; + for(int driverId = 0; driverId < (int)driverCount; driverId++) { + for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { + device_t *device = &drivers[driverId].devices[deviceId]; + if(NULL != device_workspaceA[did] && NULL != device_workspaceC[did]) { + fprintf(stderr, "STATUS: Ready to submit GEMM[%d] on device %d of driver %d\n", run, deviceId, driverId); + fprintf(stderr, "STATUS: Context of driver %d is %s\n", driverId, zeContextGetStatus(drivers[driverId].context) == ZE_RESULT_SUCCESS ? "Fine" : "Broken"); + dpcpp_kernel_GEMM(device->driver->swp, device->swd, device->streams[2].swq, (double*)device_workspaceA[did], (double*)device_workspaceC[did], N); + fprintf(stderr, "STATUS: GEMM[%d] submitted on device %d of driver %d\n", run, deviceId, driverId); + + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].start); + ze_rc = zeCommandListReset(device->streams[2].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListReset ", ze_rc, { continue; } ); + ze_rc = zeCommandListClose(device->streams[2].cl); + LEVEL_ZERO_CHECK_ERROR( "zeCommandListClose ", ze_rc, { continue; } ); + + ze_rc = zeFenceReset(device->streams[2].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeFenceReset ", ze_rc, { continue; } ); + ze_rc = zeCommandQueueExecuteCommandLists(device->streams[2].cq, 1, &device->streams[2].cl, device->streams[2].fences[0]); + LEVEL_ZERO_CHECK_ERROR( "zeCommandQueueExecuteCommandLists ", ze_rc, { continue; } ); + + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].enter_wait); + do { + ze_rc = zeFenceQueryStatus(device->streams[2].fences[0]); + if( ZE_RESULT_SUCCESS == ze_rc ) { + fprintf(stderr, "STATUS: GEMM[%d] ended on device %d of driver %d\n", run, deviceId, driverId); + break; + } else if( ZE_RESULT_NOT_READY != ze_rc ) { + LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); + } else { + usleep(10); + } + } while(1); + clock_gettime(CLOCK_REALTIME, &timings[did].runs[run].end); + fprintf(stderr, "STATUS: GEMM[%d] on device %d of driver %d completed!\n", run, deviceId, driverId); + } + did++; + } + } + } + did = 0; for(int driverId = 0; driverId < (int)driverCount; driverId++) { for(int deviceId = 0; deviceId < drivers[driverId].nb_devices; deviceId++) { @@ -472,19 +512,70 @@ int main(int argc, char *argv[]) do { ze_rc = zeFenceQueryStatus(device->streams[1].fences[0]); if( ZE_RESULT_SUCCESS == ze_rc ) { - fprintf(stderr, "STATUS: GEMM ended on device %d of driver %d\n", deviceId, driverId); + fprintf(stderr, "STATUS: copies ended from device %d of driver %d\n", deviceId, driverId); + break; } else if( ZE_RESULT_NOT_READY != ze_rc ) { LEVEL_ZERO_CHECK_ERROR( "(progress_stream) zeFenceQueryStatus ", ze_rc, { continue; } ); } else { usleep(10); } } while(1); - } else { - fprintf(stderr, "Skipping device %d which failed at allocating data\n", did); + clock_gettime(CLOCK_REALTIME, &timings[did].end_gpu2cpu); + fprintf(stderr, "STATUS: GPU->CPU copies on device %d of driver %d completed!\n", deviceId, driverId); + + int error0 = 0; + int error1 = 0; + for(int i = 0; i < N; i++) { + for(int j = 0; j < N; j++) { + if(i != j) { + if(cpuC[i*N+j] != 0.0) { + if(0 == error0) + fprintf(stderr, "ERROR during checks of driver %d: C[%d,%d] = %g expected 0.0\n", did, i, j, cpuC[i*N+j]); + error0++; + errors++; + } + } else { + if(cpuC[i*N+j] != (double)NRUNS) { + if(0 == error1) + fprintf(stderr, "ERROR during checks of driver %d: C[%d, %d] = %g, expected %g\n", did, i, j, cpuC[i*N+j], (double)NRUNS); + error1++; + errors++; + } + } + } + } + if(error0 > 0) fprintf(stderr, " %d errors of type 0 during checks of driver %d\n", error0, did); + if(error1 > 0) fprintf(stderr, " %d errors of type 1 during checks of driver %d\n", error1, did); } did++; } } - return EXIT_SUCCESS; + for(did = 0; did < nb_devices; did++) { + printf("TIME: Device %d\n", did); + printf("TIME: START: %ld.%09ld s\n", timings[did].start.tv_sec, timings[did].start.tv_nsec); + printf("TIME: START_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].start_cpu2gpu, &timings[did].start)); + printf("TIME: START_ENTER_WAIT_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].enter_wait_cpu2gpu, &timings[did].start_cpu2gpu)); + printf("TIME: END_CPU2GPU: %11.9g s\n", diff_timespec(&timings[did].end_cpu2gpu, &timings[did].enter_wait_cpu2gpu)); + for(int run = 0; run < NRUNS; run++) { + printf("TIME: START_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].start, + run == 0 ? &timings[did].end_cpu2gpu : &timings[did].runs[run-1].end)); + printf("TIME: ENTER_WAIT_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].enter_wait, &timings[did].runs[run].start)); + printf("TIME: END_KERNEL(%3d): %11.9g s\n", run, + diff_timespec(&timings[did].runs[run].end, &timings[did].runs[run].enter_wait)); + double tot_kernel = diff_timespec(&timings[did].runs[run].end, &timings[did].runs[run].start); + double gflops = 2.0*N*N*N/1e9; + printf("PERF: device %d run %d %11.9g s for %d x %d x %d DGEMM (%g . 10^9 flops): %g GFLOP/s\n", did, run, tot_kernel, N, N, N, gflops, gflops/tot_kernel); + } + printf("TIME: START_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].start_gpu2cpu, &timings[did].runs[NRUNS-1].end)); + printf("TIME: ENTER_WAIT_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].enter_wait_gpu2cpu, &timings[did].start_gpu2cpu)); + printf("TIME: END_GPU2CPU: %11.9g s\n", diff_timespec(&timings[did].end_gpu2cpu, &timings[did].enter_wait_gpu2cpu)); + printf("\n"); + } + + if(errors == 0) + return EXIT_SUCCESS; + return EXIT_FAILURE; }