diff --git a/.github/workflows/RunnerFastCI.yml b/.github/workflows/RunnerFastCI.yml new file mode 100644 index 0000000000..d0f1c6c5f4 --- /dev/null +++ b/.github/workflows/RunnerFastCI.yml @@ -0,0 +1,52 @@ +# This is a basic workflow to help you get started with Actions +name: FastCI4Push + +defaults: + run: + shell: bash + +# Controls when the action will run. Triggers the workflow on pull request +# events but only for the master branch +on: [push, fork] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + FastCI4Push: + # The type of runner that the job will run on + runs-on: [self-hosted, linux, x64] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v4 + with: + clean: true + + # Configure-Build-Test + - name: Compile-Test NHVHPC + run: | + ls -ltr ~/.bashrc + source /etc/profile.d/lmod.sh + source ~/.bash_aliases + module load nvhpc + export CXX=mpicxx + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=./build/opt -DCMAKE_BUILD_TYPE=Release -DLEGACY_CODEGEN=OFF -DHDF5_ROOT=/usr/lib/x86_64-linux-gnu/hdf5/openmpi + cmake --build build -j 12 + cmake --install build + #ctest --test-dir build --output-on-failure + rm -rf build + + # Configure-Build-Test + - name: Compile-Test GCC13 + run: | + ls -ltr ~/.bashrc + source /etc/profile.d/lmod.sh + source ~/.bash_aliases + module load OpenMPI + module load CUDA + export CXX=mpicxx + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=./build/opt -DCMAKE_CUDA_HOST_COMPILER=gcc-11 -DCMAKE_BUILD_TYPE=Release -DLEGACY_CODEGEN=OFF -DHDF5_ROOT=/usr/lib/x86_64-linux-gnu/hdf5/openmpi + cmake --build build -j 8 + cmake --install build + ctest --test-dir build --output-on-failure + rm -rf build diff --git a/.gitignore b/.gitignore index 2ccbeb9577..3499ca437d 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,7 @@ ops/fortran/mod *.lst tea*.in .clang-format +build* +Build* +BUILD* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml_nouse similarity index 81% rename from .gitlab-ci.yml rename to .gitlab-ci.yml_nouse index 723f1a8647..3af9cfa08f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml_nouse @@ -1,13 +1,9 @@ -# Defines stages which are to be executed stages: - - API_CXX_Build - - API_FORTRAN_Build - - APPS_CXX_Build_and_Test - - APPS_FORTRAN_Build_and_Test + - test_gnu11 + - test_nvhpc239 -# Stage "API_Build" -build:GNU11:API_CXX: - stage: API_CXX_Build +build_test_gnu11: + stage: test_gnu11 tags: - CCP, test script: @@ -15,58 +11,26 @@ build:GNU11:API_CXX: - module avail - module load CUDA - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DGPU_ARCH=75 - - cmake --build . -j 4 - - cmake --install . - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-INSTALL + #- cmake -S . -B build -DCMAKE_INSTALL_PREFIX=./build/opt -DOPS_TEST_FULL=OFF + - cmake -S . -B build -DCMAKE_INSTALL_PREFIX=./build/opt -DOPS_TEST=OFF + - cmake --build build -j 6 + - cmake --install build + - echo `pwd` + - echo $CI_PROJECT_DIR + - ctest --test-dir build --output-on-failure + #- mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts + #- ssh-keyscan -H bitbucket.org >> ~/.ssh/known_hosts + #- echo -e "machine bitbucket.org\nlogin gitlab-ci-token\npassword ${CI_JOB_TOKEN}" > ~/.netrc + - git config --global --add safe.directory /home/rfj82982/GIT/opensbli_jaxa.git + - git clone /home/rfj82982/GIT/opensbli_jaxa + - cd opensbli_jaxa + - git checkout UpdateTests + - git status + - ls $CI_PROJECT_DIR/build/opt + - ./RunTests.sh /usr/lib/x86_64-linux-gnu/hdf5/openmpi $CI_PROJECT_DIR/build/opt -build:GNU11:API_FORTRAN: - stage: API_FORTRAN_Build - tags: - - CCP, test - script: - - source ~/.bash_aliases - - module avail - - module load CUDA - - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DBUILD_OPS_CXX=OFF -DBUILD_OPS_FORTRAN=ON -DGPU_ARCH=75 - - cmake --build . - - cmake --install . - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-INSTALL - -# This is the latest easybuild based on GNU -build:foss:API_CXX: - stage: API_CXX_Build - tags: - - CCP, test - script: - - source ~/.bash_aliases - - module avail - - module load foss CUDA - - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DGPU_ARCH=75 -DCMAKE_CUDA_HOST_COMPILER=gcc-11 - - cmake --build . -j 4 - - cmake --install . - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-INSTALL - -# Build with NVHPC compiler -build:NVHCP23:API_CXX: - stage: API_CXX_Build +build_test_nvhpc239: + stage: test_nvhpc239 tags: - CCP, test script: @@ -74,120 +38,8 @@ build:NVHCP23:API_CXX: - module avail - module load nvhpc/23.9 - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DGPU_ARCH=75 - - cmake --build . -j 4 - - cmake --install . - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-INSTALL - -# Build with NVHPC compiler -build:NVHCP23:API_FORTRAN: - stage: API_FORTRAN_Build - tags: - - CCP, test - script: - - source ~/.bash_aliases - - module avail - - module load nvhpc/23.9 - - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DBUILD_OPS_CXX=OFF -DBUILD_OPS_FORTRAN=ON -DGPU_ARCH=75 - - cmake --build . - - cmake --install . - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-INSTALL - -# Stage "Test" -build_and_test:GNU11:APPS_CXX: - stage: APPS_CXX_Build_and_Test - tags: - - CCP, test - script: - - source ~/.bash_aliases - - module avail - - module load CUDA - - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DBUILD_OPS_APPS=ON -DAPP_INSTALL_DIR=$CI_PROJECT_DIR/OPS-APP -DOPS_TEST=ON -DGPU_ARCH=75 -DGPU_NUMBER=1 - - cmake --build . -j 4 - - cmake --install . -# - ctest -V - - ctest - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-APP - -build_and_test:GNU11:APPS_FORTRAN: - stage: APPS_FORTRAN_Build_and_Test - tags: - - CCP, test - script: - - source ~/.bash_aliases - - module avail - - module load CUDA - - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DBUILD_OPS_CXX=OFF -DBUILD_OPS_FORTRAN=ON -DBUILD_OPS_APPS=ON -DAPP_INSTALL_DIR=$CI_PROJECT_DIR/OPS-APP -DOPS_TEST=ON -DGPU_ARCH=75 -DGPU_NUMBER=1 - - cmake --build . - - cmake --install . -# - ctest -V - - ctest - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-APP - -build_and_test:NVHCP23:APPS_FORTRAN: - stage: APPS_FORTRAN_Build_and_Test - tags: - - CCP, test - script: - - source ~/.bash_aliases - - module avail - - module load nvhpc/23.9 - - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DBUILD_OPS_CXX=OFF -DBUILD_OPS_FORTRAN=ON -DBUILD_OPS_APPS=ON -DAPP_INSTALL_DIR=$CI_PROJECT_DIR/OPS-APP -DOPS_TEST=ON -DGPU_ARCH=75 -DGPU_NUMBER=1 - - cmake --build . --verbose - - cmake --install . -# - ctest -V - - ctest - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-INSTALL - -build_and_test:foss:APPS_CXX: - stage: APPS_CXX_Build_and_Test - tags: - - CCP, test - script: - - source ~/.bash_aliases - - module avail - - module load foss CUDA - - export CXX=mpicxx - - mkdir build - - cd build - - cmake ../ -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/OPS-INSTALL -DBUILD_OPS_APPS=ON -DAPP_INSTALL_DIR=$CI_PROJECT_DIR/OPS-APP -DOPS_TEST=ON -DGPU_ARCH=75 -DCMAKE_CUDA_HOST_COMPILER=gcc-11 -DGPU_NUMBER=1 - - cmake --build . -j 4 - - cmake --install . -# - ctest -V - - ctest - artifacts: - expire_in: 1 week - paths: - - $CI_PROJECT_DIR/OPS-APP + #- cmake -S . -B build -DCMAKE_INSTALL_PREFIX=./build/opt -DOPS_TEST_FULL=ON -DOPS_TEST_DEV=ON -DHDF5_ROOT=/usr/lib/x86_64-linux-gnu/hdf5/openmpi/ + #- cmake --build build -j 6 # Build and Test with NVHPC compiler #build_and_test:NVHCP23:APPS: diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d3d14fdf7..13071180a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,333 +1,76 @@ -# Specifiy the cmake version range, currenty work under 3.18 - cmake_minimum_required(VERSION 3.18) - - project(OPS C CXX Fortran) - -# if show the compiling process in detail - option(OPS_VERBOSE_WARNING "Turn on verbose warning messages" OFF) - option(OPS_TEST "Turn on tests for Apps" OFF) - option(OPS_HIP "Turn on the HIP backend" OFF) - option(ENABLE_IEEE "Enable IEEE flags" OFF) - option(LEGACY_CODEGEN "Build apps with Legacy code-gen" OFF) - option(BUILD_OPS_CXX "Build CXX version of OPS" ON) - option(BUILD_OPS_FORTRAN "Build FORTRAN version of OPS" OFF) - option(BUILD_OPS_APPS "Build Sample OPS applications" OFF) - - if(NOT OPS_VERBOSE_WARNING) - message( - "We show concise compiling information by default \n! Use -DOPS_VERBOSE_WARNING=ON to switch on." - ) - endif() - - set(CMAKE_VERBOSE_MAKEFILE ${OPS_VERBOSE_WARNING}) - -# Use the Release mode by default - if(NOT CMAKE_BUILD_TYPE) - message("Using default mode. Please use -DCMAKE_BUILD_TYPE to change to Release or Debug.") - set(CMAKE_BUILD_TYPE "") - endif() - - message( - WARNING - "The installation directory is ${CMAKE_INSTALL_PREFIX} \n! Use -DCMAKE_INSTALL_PREFIX to change" - ) - -# Tridiagonal library - if(NOT LIBTRID_PATH) - message( - WARNING - "The install path of the scalar tridiagonal solver library (LIBTRID_PATH) is not specified and it will not compiled!" - ) - endif() - - -# Configure C Compiler - set(CMAKE_C_STANDARD 99) - - if(${CMAKE_C_COMPILER_ID} STREQUAL GNU) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -std=c99 -fPIC -Wall -ffloat-store -g -ftree-vectorize -fopenmp") - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -std=c99 -fPIC -Wall -ffloat-store -g -ftree-vectorize -fopenmp") - set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -std=c99 -fPIC -Wall -ffloat-store -g -fopenmp") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL Clang) - set(CMAKE_C_FLAGS "-fPIC -DUNIX -Wall") - set(CMAKE_C_FLAGS_RELEASE "-O3") - set(CMAKE_C_FLAGS_DEBUG "-g -O0") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL MSVC) - set(CMAKE_C_FLAGS "-W1") - set(CMAKE_C_FLAGS_RELEASE "/O2 /Ot ") - set(CMAKE_C_FLAGS_DEBUG "/Z7 /DEBUG") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -fast -gopt") - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -fast -gopt") - set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL Intel) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -g -xHost -std=c99 -qopenmp -qopt-report") - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -g -xHost -std=c99 -qopenmp -qopt-report") - set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -xHost -g -std=c99") - else() - message( - WARNING - "There is no default flag for the chosen C compiler! Please use -DCFLAG=XXX to supply!" - ) - endif() - -# Configure C++ Compiler - set(CMAKE_CXX_STANDARD 11) - - if(${CMAKE_CXX_COMPILER_ID} STREQUAL GNU) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -Wall -ffloat-store -g -std=c++11 -fopenmp") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -fPIC -Wall -ffloat-store -g -std=c++11 -fopenmp") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fPIC -Wall -ffloat-store -g -std=c++11") - elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL Clang) - set(CMAKE_CXX_FLAGS "-fPIC -DUNIX -Wall") - set(CMAKE_CXX_FLAGS_RELEASE "-O3") - set(CMAKE_CXX_FLAGS_DEBUG "-g -O0") - elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC) - set(CMAKE_CXX_FLAGS "-W1") - set(CMAKE_CXX_FLAGS_RELEASE "/O2 /Ot ") - set(CMAKE_CXX_FLAGS_DEBUG "/Z7 /DEBUG") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fast -gopt -std=c++11 -mp") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -fast -gopt -std=c++11 -mp") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g -std=c++11 -mp") - elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL Intel) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -xHost -qopenmp -qopt-report -std=c++11 -qopt-zmm-usage=high") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -g -xHost -qopenmp -qopt-report -std=c++11 -qopt-zmm-usage=high") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g -xHost -DOPS_DEBUG -std=c++11 -DOPS_DEBUG") - else() - message( - WARNING - "There is no default flag for the chosen C++ compiler! Please use -DCXXFLAG=XXX to supply!" - ) - endif() - - -# Adding CFLAGS provided to CMAKE_C_FLAGS - if(CFLAG) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CFLAG}") - else() - message( - "Extra C compiler options can be provided using CFLAG, e.g., -DCFLAG=\"-ffloat-store\" for GNU!" - ) - endif() - -# Adding CXXFLAGS provided to CMAKE_CXX_FLAGS - if(CXXFLAG) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXXFLAG}") - else() - message( - "Extra CXX compiler options can be provided using CXXFLAG, e.g., -DCXXFLAG=\"-ffloat-store\" for GNU!" - ) - endif() - -# If IEEE is enabled - if(ENABLE_IEEE) -# Add IEEE flags to CMAKE_CXX_FLAGS - if(${CMAKE_CXX_COMPILER_ID} STREQUAL Intel) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fp-model strict -fp-model source -prec-div -prec-sqrt") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fp-model strict -fp-model source -prec-div -prec-sqrt") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Kieee -nofma") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Kieee -nofma") - else() - message( - WARNING - "There is no default IEEE flags for chosen compiler" - ) - endif() - endif() - - if(CMAKE_BUILD_TYPE STREQUAL "Release") - message(STATUS "CMAKE_C_FLAGS_RELEASE: ${CMAKE_C_FLAGS_RELEASE}") - message(STATUS "CMAKE_CXX_FLAGS_RELEASE: ${CMAKE_CXX_FLAGS_RELEASE}") - elseif(CMAKE_BUILD_TYPE STREQUAL "Debug") - message(STATUS "CMAKE_C_FLAGS_DEBUG: ${CMAKE_C_FLAGS_DEBUG}") - message(STATUS "CMAKE_CXX_FLAGS_DEBUG: ${CMAKE_CXX_FLAGS_DEBUG}") - elseif(CMAKE_BUILD_TYPE STREQUAL "") - message(STATUS "CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") - message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") - else() - message(STATUS "Unknown build type: ${CMAKE_BUILD_TYPE}") - endif() - -# include(GenerateExportHeader) - include(CMakePackageConfigHelpers) - - set(VERSION latest) - - set(ConfigPackageLocation ${CMAKE_INSTALL_PREFIX}/lib/cmake) - -# if (CFLAG) add_compile_options(${CFLAG}) endif () Add find scripts - list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) - - set(HDF5_PREFER_PARALLEL true) - -# Try to find the required dependency - find_package(MPI QUIET) - find_package(HDF5 QUIET COMPONENTS C HL) - find_package(CUDAToolkit QUIET) - find_package(OpenMP REQUIRED) - - find_package(Python3 3.8 REQUIRED) - if(NOT Python3_FOUND) - message( - WARNING - "We cannot find Python3 and the OPS code-generator needs Python3! Please use -DPython3_EXECUTABLE to specify the path." - ) - endif() - - if(OPS_HIP) -# Search for rocm in common locations - list(APPEND CMAKE_PREFIX_PATH /opt/rocm/hip /opt/rocm) - -# Find hip - find_package(hip) - -# enable_language(HIP) message($ENV{HIP_PLATFORM}) - - set(ENV{HIP_PLATFORM} "nvidia") -# message($ENV{HIP_PLATFORM}) - - set(HIP_FOUND TRUE) - - if({$hip::host} STREQUAL "" OR {$hip::device} STREQUAL "") - message( - WARNING - "We cannot find the HIP environment. The HIP codes won't work! You might need to use CMAKE_PREFIX_PATH to specify the path for HIP!" - ) - set(HIP_FOUND FALSE) - endif() - endif() - -# Configure the "include" dir for compiling - if(NOT HDF5_FOUND) - message( - WARNING - "We cannot find the parallel HDF5 library. The HDF5 IO routines won't work! Please use -DHDF5_ROOT to specify the path!" - ) - endif() - - if(NOT MPI_FOUND) - message( - WARNING "We cannot find a MPI environment! The MPI codes won't compile!") - endif() - - if(LIBTRID_PATH) - include_directories(${LIBTRID_PATH}/include) - endif() - - if(CUDAToolkit_FOUND) - set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE}) - if(GPU_ARCH) - set(CMAKE_CUDA_ARCHITECTURES - ${GPU_ARCH} - CACHE STRING "CUDA architectures") - else() - set(CMAKE_CUDA_ARCHITECTURES - 70 - CACHE STRING "CUDA architectures") - message(WARNING "Using Volta architecure for compiling CUDA. \n If different please provide correct GPU architecture using -DGPU_ARCH=XXX where XXX is compute capability of an architecture \n For ex. -DGPU_ARCH=70 if you want to target sm_70") - endif() - - enable_language(CUDA) - - else() - message(WARNING "We cannot find a CUDA toolkit! The CUDA codes won't compile!") - endif() - - if(USE_NCCL) - find_package(NCCL) - if(NOT NCCL_FOUND) - message( - WARNING - "We cannot find NCCL! Tridiagonal library wrapper will be compiled with NCCL" - ) - endif() - endif() - - -# option(OPS_ERROR_ON_WARNING "Error upon encountering a warning" ON) -# option(CHECKPOINTING "Enable OPS checkpointing support" OFF) -# option(OPS_INSTALL_TRANSLATOR "Install OPS Source Translator" ON) -# option(OPS_INSTALL_HEADERS "Install OPS Dev Headers" ON) -# option(OPS_INSTALL_LIBS "Install OPS Libraries" ON) - -# Turn on all the warnings if(OPS_VERBOSE_WARNINGS) if (WIN32) list(APPEND -# OPS_WARNINGS /W4) list(APPEND OPS_WARNINGS /wd4849) list(APPEND OPS_WARNINGS -# /wd4127) list(APPEND OPS_WARNINGS /D_CRT_SECURE_NO_WARNINGS) else() -# list(APPEND OPS_WARNINGS -Wall -Wextra -pedantic -Wno-unused-function -# -Wno-unused-parameter) endif() foreach(flag IN LISTS OPS_WARNINGS) # NVCC -# complicates life a little here # 1) We want to pass our warning flags to the -# compiler, not NVCC, so # each need prefaced with -Xcompiler # 2) NVCC -# generates code which is not pedantically compliant, so need # to filter the -# -pedantic from compiling cuda code -# add_compile_options("$,$<$>:SHELL:-Xcompiler -# ${flag}>,${flag}>") endforeach() endif() if(OPS_ERROR_ON_WARNING) if (WIN32) -# set(werrorFlag "/WX") else() set(werrorFlag "-Werror") endif() -# add_compile_options("$<$:SHELL:-Xcompiler -# >${werrorFlag}") endif() - -# install python translator - if(LEGACY_CODEGEN) - add_subdirectory(ops_translator_legacy) - else() - add_subdirectory(ops_translator) - endif() - -# Configure the C/C++ library compiling - if(BUILD_OPS_CXX) - add_subdirectory(ops/c) - endif() - -# Configure the installation of "include" - file(GLOB_RECURSE INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/ops/c/include/*.h" - "${CMAKE_CURRENT_SOURCE_DIR}/ops/fortran/include/*.inc") - - install(FILES ${INCLUDE} DESTINATION include) - -# prepare the script for setting OPS environment variable - if(UNIX) - install( - CODE "file(WRITE \"${CMAKE_INSTALL_PREFIX}/bin/ops_env\" \"export OPS_INSTALL_PATH=${CMAKE_INSTALL_PREFIX}\n\")" - ) - - if(NOT ((CMAKE_INSTALL_PREFIX STREQUAL "/usr") - OR (CMAKE_INSTALL_PREFIX STREQUAL "/usr/local"))) - install( - CODE "file(APPEND \"${CMAKE_INSTALL_PREFIX}/bin/ops_env\" \"export PATH=$PATH:${CMAKE_INSTALL_PREFIX}/bin\n\")" - ) - endif() - endif() - - if(BUILD_OPS_FORTRAN) - add_subdirectory(ops/fortran) - endif() - -# prepare ctest - if(OPS_TEST) - enable_testing() - include(ProcessorCount) - ProcessorCount(CPU_NUMBER) - if(CPU_NUMBER GREATER 12) - set(CPU_NUMBER 12) - endif() - if(NOT GPU_NUMBER) - message( - WARNING "GPU tests are disabled! Please use -DGPU_NUMBER to specify!") - endif() - endif() - - if(BUILD_OPS_CXX) - install(FILES "${CMAKE_BINARY_DIR}/OPSConfig.cmake" - DESTINATION ${ConfigPackageLocation}) - endif() - - if(BUILD_OPS_APPS) - if(BUILD_OPS_CXX) - add_subdirectory(apps/c) - endif() - if(BUILD_OPS_FORTRAN) - add_subdirectory(apps/fortran) - endif() - endif() - +# To specifiy the cmake version range Currenty work under 3.18 +cmake_minimum_required(VERSION 3.18) +project(OPS CXX) +enable_testing() + +# if show the compiling process in detail +option(OPS_TEST "Turn on tests for Apps" ON) +option(OPS_HIP "Turn on the HIP backend" OFF) +option(OPS_CUDA "Turn on the CUDA backend" ON) +option(OPS_FORTRAN "Turn on the Fortran build" OFF) +option(IEEE "Use IEEE flags" ON) + +# Output of the installation directory +include(GNUInstallDirs) +set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}) +set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}) +set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}) +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/opt" CACHE PATH "..." FORCE) +endif() +message(STATUS + "The installation directory is ${CMAKE_INSTALL_PREFIX}! Use -DCMAKE_INSTALL_PREFIX to change" +) + +# Use the Release mode by default +if(NOT CMAKE_BUILD_TYPE) + message(STATUS + "BUILD_TYPE is Release mode! Please uses -DCMAKE_BUILD_TYPE to change.") + set (CMAKE_BUILD_TYPE RELEASE CACHE STRING + "Choose the type of build, options are: None Dev Debug Release." + FORCE) +endif(NOT CMAKE_BUILD_TYPE) + + +# Set the verbose level +option(OPS_VERBOSE_WARNING "Turn on verbose warning messages" OFF) +if(NOT OPS_VERBOSE_WARNING) + message(STATUS + "Verbose output for compilation is set to concise! Use -DOPS_VERBOSE_WARNING=ON to increase verbose." + ) +endif() +set(CMAKE_VERBOSE_MAKEFILE ${OPS_VERBOSE_WARNING}) + +## Tridiagonal library +#if(NOT LIBTRID_PATH) +# message(WARNING +# "The install path of the scalar tridiagonal solver library (LIBTRID_PATH) is not specified and it will not be compiled!" +# ) +#endif() + + +# Try to find the required dependency +# Add support for CMAKE_DEPENDENT_OPTION +INCLUDE(CMakeDependentOption) +INCLUDE(CMakeParseArguments) +include(CMakePackageConfigHelpers) +set(VERSION latest) +set(ConfigPackageLocation ${CMAKE_INSTALL_PREFIX}/lib/cmake) + +# Include paths to find pachages +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake" "${CMAKE_SOURCE_DIR}/cmake/compilers" "${CMAKE_SOURCE_DIR}/cmake/packages") + +# Activate Fortran if necessary and check if provided +include(OPS_Fortran) + +include(OPS_FindPackages) +include(OPS_Compilers) +# install python translator +if(Python3_FOUND) + add_subdirectory(ops_translator) +endif() +add_subdirectory(ops) +# Build the APPS and TEST +if(OPS_TEST) + add_subdirectory(apps) +endif() diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt new file mode 100644 index 0000000000..dc962f4b74 --- /dev/null +++ b/apps/CMakeLists.txt @@ -0,0 +1,45 @@ +# Create a folder directory for testing +set(app_dir "${PROJECT_BINARY_DIR}/APPS_BUILD") +file(MAKE_DIRECTORY ${app_dir}) +# Activate the full list of tests +option(OPS_TEST_FULL "Turn on tests for all Apps" OFF) +option(OPS_TEST_DEV "Turn on tests for dev backends" OFF) + +# Find the OPS Translator +if(LEGACY_CODEGEN) + find_path(opsc + NAMES "ops.py" + PATHS ${SEARCH_PATHS} + PATH_SUFFIXES translator/ops_translator_legacy/c) + + find_path(opsfortran + NAMES "ops_fortran.py" + PATHS ${SEARCH_PATHS} + PATH_SUFFIXES translator/ops_translator_legacy/fortran) +else() + find_path(opsc + NAMES "__main__.py" + PATHS ${SEARCH_PATHS} + PATH_SUFFIXES translator/ops_translator/ops-translator) + + find_path(opsfortran + NAMES "__main__.py" + PATHS ${SEARCH_PATHS} + PATH_SUFFIXES translator/ops_translator/ops-translator) +endif() + +if(NOT opsc) + message(FATAL_ERROR "We cannot find the python C/C++ translator!") +endif() + +if(NOT opsfortran) + message(FATAL_ERROR "We cannot find the python fortran translator!") +endif() + +# Add include and link directories +#${CMAKE_INSTALL_PREFIX}/lib +#include_directories("${CMAKE_INSTALL_PREFIX}/include") +#message(STATUS "SOURCE DIR ${CMAKE_SOURCE_DIR}") +#message(STATUS "INSTALL PREFIX ${CMAKE_INSTALL_PREFIX}") +link_directories("${PROJECT_BINARY_DIR}/lib") +add_subdirectory(c) diff --git a/apps/c/CMakeLists.txt b/apps/c/CMakeLists.txt index 3f5a81c3fb..5e6de28976 100644 --- a/apps/c/CMakeLists.txt +++ b/apps/c/CMakeLists.txt @@ -1,1080 +1,54 @@ -# Specifiy the cmake version range, currently work under 3.18 - cmake_minimum_required(VERSION 3.18) - - project(APP C CXX) - - if(NOT APP_INSTALL_DIR) - set(APP_INSTALL_DIR "$ENV{HOME}/OPS-APPS") - message( - WARNING - "We will install the applications to ${APP_INSTALL_DIR} \n! Please use -DAPP_INSTALL_DIR to specify!" - ) - endif() - -# If APPs are compiled independently then setup compilers and dependencies. - if(${CMAKE_PROJECT_NAME} STREQUAL APP) -# Show the compiling process in detail - option(OPS_VERBOSE_WARNING "Turn on verbose warning messages" OFF) - option(OPS_TEST "Turn on tests for Apps" OFF) - option(ENABLE_IEEE "Enable IEEE flags" OFF) - option(LEGACY_CODEGEN "Build apps with Legacy code-gen" OFF) - - if(NOT OPS_VERBOSE_WARNING) - message( - "We show concise compiling information by default \n! Use -DOPS_VERBOSE_WARNING=ON to switch on." - ) - endif() - - set(CMAKE_VERBOSE_MAKEFILE ${OPS_VERBOSE_WARNING}) - set(OPS_APP_SRC ${CMAKE_SOURCE_DIR}) - - if(NOT CMAKE_BUILD_TYPE) - message("Using default mode. Please use -DCMAKE_BUILD_TYPE to change to Release or Debug.") - set(CMAKE_BUILD_TYPE "") - endif() - - if(NOT LIBTRID_PATH) - message( - WARNING - "The install path of the scalar tridiagonal solver library (LIBTRID_PATH) is not specified and the apps which need it will not compiled!" - ) - endif() - -# Configure C Compiler - set(CMAKE_C_STANDARD 99) - - if(${CMAKE_C_COMPILER_ID} STREQUAL GNU) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -std=c99 -fPIC -Wall -ffloat-store -g -ftree-vectorize -fopenmp") - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -std=c99 -fPIC -Wall -ffloat-store -g -ftree-vectorize -fopenmp") - set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -std=c99 -fPIC -Wall -ffloat-store -g -fopenmp") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL Clang) - set(CMAKE_C_FLAGS "-fPIC -DUNIX -Wall") - set(CMAKE_C_FLAGS_RELEASE "-O3") - set(CMAKE_C_FLAGS_DEBUG "-g -O0") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL MSVC) - set(CMAKE_C_FLAGS "-W1") - set(CMAKE_C_FLAGS_RELEASE "/O2 /Ot ") - set(CMAKE_C_FLAGS_DEBUG "/Z7 /DEBUG") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -fast -gopt") - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -fast -gopt") - set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL Intel) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -g -xHost -std=c99 -qopenmp -qopt-report") - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -g -xHost -std=c99 -qopenmp -qopt-report") - set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -xHost -g -std=c99") - else() - message( - WARNING - "There is no default flag for the chosen C compiler! Please use -DCFLAG=XXX to supply!" - ) - endif() - -# Configure CXX Compiler - set(CMAKE_CXX_STANDARD 11) - - if(${CMAKE_CXX_COMPILER_ID} STREQUAL GNU) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -Wall -ffloat-store -g -std=c++11 -fopenmp") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -fPIC -Wall -ffloat-store -g -std=c++11 -fopenmp") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fPIC -Wall -ffloat-store -g -std=c++11") - elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL Clang) - set(CMAKE_CXX_FLAGS "-fPIC -DUNIX -Wall") - set(CMAKE_CXX_FLAGS_RELEASE "-O3") - set(CMAKE_CXX_FLAGS_DEBUG "-g -O0") - elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC) - set(CMAKE_CXX_FLAGS "-W1") - set(CMAKE_CXX_FLAGS_RELEASE "/O2 /Ot ") - set(CMAKE_CXX_FLAGS_DEBUG "/Z7 /DEBUG") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fast -gopt -std=c++11 -mp") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -fast -gopt -std=c++11 -mp") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g -std=c++11 -mp") - elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL Intel) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -xHost -qopenmp -qopt-report -std=c++11 -qopt-zmm-usage=high") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -g -xHost -qopenmp -qopt-report -std=c++11 -qopt-zmm-usage=high") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g -xHost -DOPS_DEBUG -std=c++11 -DOPS_DEBUG") - else() - message( - WARNING - "There is no default flag for the chosen C++ compiler! Please use -DCXXFLAG=XXX to supply!" - ) - endif() - - -# Adding CFLAGS provided to CMAKE_C_FLAGS - if(CFLAG) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CFLAG}") - else() - message( - "Extra C compiler options can be provided using CFLAG, e.g., -DCFLAG=\"-ffloat-store\" for GNU!" - ) - endif() - -# Adding CXXFLAGS provided to CMAKE_CXX_FLAGS - if(CXXFLAG) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXXFLAG}") - else() - message( - "Extra CXX compiler options can be provided using CXXFLAG, e.g., -DCXXFLAG=\"-ffloat-store\" for GNU!" - ) - endif() - -# If IEEE is enabled - if(ENABLE_IEEE) -# Add IEEE flags to CMAKE_CXX_FLAGS - if(${CMAKE_CXX_COMPILER_ID} STREQUAL Intel) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fp-model strict -fp-model source -prec-div -prec-sqrt") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fp-model strict -fp-model source -prec-div -prec-sqrt") - elseif(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Kieee -nofma") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Kieee -nofma") - else() - message( - WARNING - "There is no default IEEE flags for chosen compiler" - ) - endif() - endif() - - if(CMAKE_BUILD_TYPE STREQUAL "Release") - message(STATUS "CMAKE_C_FLAGS_RELEASE: ${CMAKE_C_FLAGS_RELEASE}") - message(STATUS "CMAKE_CXX_FLAGS_RELEASE: ${CMAKE_CXX_FLAGS_RELEASE}") - elseif(CMAKE_BUILD_TYPE STREQUAL "Debug") - message(STATUS "CMAKE_C_FLAGS_DEBUG: ${CMAKE_C_FLAGS_DEBUG}") - message(STATUS "CMAKE_CXX_FLAGS_DEBUG: ${CMAKE_CXX_FLAGS_DEBUG}") - elseif(CMAKE_BUILD_TYPE STREQUAL "") - message(STATUS "CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") - message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") - else() - message(STATUS "Unknown build type: ${CMAKE_BUILD_TYPE}") - endif() - - -# Trying to find the OPS library TODO this shall become FindOPS.cmake later - set(SEARCH_PATHS "${OPS_INSTALL_DIR}" "/usr/local" "/usr") -# Tyring to find the include directory - set(OPS_HEAD - ops_checkpointing.h - ops_cuda_reduction.h - ops_cuda_rt_support.h - ops_device_rt_support.h - ops_exceptions.h - ops_hdf5.h - ops_hdf5_common.h - ops_hip_reduction.h - ops_hip_rt_support.h - ops_instance.h - ops_internal1.h - ops_internal2.h - ops_lib_core.h - ops_macros.h - ops_mpi_core.h - ops_seq.h - ops_seq_v2.h - ops_sycl_reduction.h - ops_sycl_rt_support.h - ops_tridiag.h - ops_util.h - queue.h) - - set(OPS_INCLUDE_DIR "") - foreach(head ${OPS_HEAD}) - unset(head_inc CACHE) - find_path( - head_inc - NAMES ${head} - PATHS ${SEARCH_PATHS} - PATH_SUFFIXES include) - - if(head_inc) - list(APPEND OPS_INCLUDE_DIR ${head_inc}) - else() - message(FATAL_ERROR "We cannot find ${head}") - endif() - endforeach() - - list(REMOVE_DUPLICATES OPS_INCLUDE_DIR) - include_directories("${OPS_INCLUDE_DIR}") - -# Trying to find the library directory - if(LIBTRID_PATH) - set(OPS_LIB - ops_cuda - ops_mpi_cuda - ops_hdf5_seq - ops_hdf5_mpi - ops_seq - ops_mpi - ops_trid_seq - ops_trid_mpi - ops_trid_cuda - ops_trid_mpi_cuda) - else() - set(OPS_LIB - ops_ompoffload - ops_mpi_ompoffload - ops_cuda - ops_mpi_cuda - ops_hdf5_seq - ops_hdf5_mpi - ops_seq - ops_mpi) - endif() - - set(OPS_LIBRARY_DIR "") - foreach(library ${OPS_LIB}) - find_library( - ${library}_FOUND - NAMES ${library} - PATHS ${SEARCH_PATHS} - PATH_SUFFIXES lib) - if(${library}_FOUND) - unset(dir CACHE) - get_filename_component(dir ${${library}_FOUND} DIRECTORY) - list(APPEND OPS_LIBRARY_DIR ${dir}) - else() - message("We cannot find ${library}") - endif() - endforeach() - - list(REMOVE_DUPLICATES OPS_LIBRARY_DIR) - link_directories("${OPS_LIBRARY_DIR}") - -# Set the OPS_INSTALL_PATH for testing - set(OPS_INSTALL_PATH "${OPS_INSTALL_DIR}") - set(OPS_CMAKE_INSTALL_PATH "${OPS_INSTALL_DIR}") - -# Try to find the required dependency - find_package(MPI QUIET) - find_package(HDF5 QUIET COMPONENTS C HL) - find_package(CUDAToolkit QUIET) - find_package(OpenMP QUIET) - - find_package(Python3 3.8 REQUIRED) - if(NOT Python3_FOUND) - message( - WARNING - "We cannot find Python3.8 or higher and the OPS code-generator needs Python3! Please use -DPython3_EXECUTABLE to specify the path." - ) - else() - if(LEGACY_CODEGEN) - find_path(opsc - NAMES "ops.py" - PATHS ${SEARCH_PATHS} - PATH_SUFFIXES translator/ops_translator_legacy/c) - - find_path(opsfortran - NAMES "ops_fortran.py" - PATHS ${SEARCH_PATHS} - PATH_SUFFIXES translator/ops_translator_legacy/fortran) - else() - find_path(opsc - NAMES "__main__.py" - PATHS ${SEARCH_PATHS} - PATH_SUFFIXES translator/ops_translator/ops-translator) - - find_path(opsfortran - NAMES "__main__.py" - PATHS ${SEARCH_PATHS} - PATH_SUFFIXES translator/ops_translator/ops-translator) - endif() - - if(NOT opsc) - message(FATAL_ERROR "We cannot find the python C/C++ translator!") - endif() - - if(NOT opsfortran) - message(FATAL_ERROR "We cannot find the python fortran translator!") - endif() - endif() - - if(LEGACY_CODEGEN) - set(OPS_C_TRANSLATOR "${opsc}/ops.py") - set(OPS_F_TRANSLATOR "${opsfortran}/ops_fortran.py") - else() - set(OPS_C_TRANSLATOR "${opsc}") - set(OPS_F_TRANSLATOR "${opsfortran}") - endif() - -# Configure the "include" dir for compiling - if(NOT HDF5_FOUND) - message( - WARNING - "We cannot find the parallel HDF5 library. The HDF5 IO routines won't work! Please use -DHDF5_ROOT to specify the path!" - ) - endif() - - if(NOT MPI_FOUND) - message( - WARNING "We cannot find a MPI environment! The MPI codes won't compile!") - endif() - - if(LIBTRID_PATH) - include_directories(${LIBTRID_PATH}/include) - endif() - - if(CUDAToolkit_FOUND) - set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE}) - if(GPU_ARCH) - set(CMAKE_CUDA_ARCHITECTURES - ${GPU_ARCH} - CACHE STRING "CUDA architectures") - else() - set(CMAKE_CUDA_ARCHITECTURES - 70 - CACHE STRING "CUDA architectures") - message(WARNING "Using Volta architecure for compiling CUDA. \n If different please provide correct GPU architecture using -DGPU_ARCH=XXX where XXX is compute capability of an architecture \n For ex. -DGPU_ARCH=70 if you want to target sm_70") - endif() - - enable_language(CUDA) - - else() - message(WARNING "We cannot find a CUDA toolkit! The CUDA codes won't compile!") - endif() - - if(USE_NCCL) - find_package(NCCL) - if(NOT NCCL_FOUND) - message( - WARNING - "We cannot find NCCL! Tridiagonal library wrapper will be compiled with NCCL" - ) - endif() - endif() - - set(HDF5_SEQ ${HDF5_FOUND}) - - if(ops_cuda_FOUND) - set(CUDA ${CUDAToolkit_FOUND}) - if(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(OMPOFFLOAD TRUE) - set(OMPOFFLOAD_FLAGS -Minline=reshape -Mvect=levels:5 -mp=gpu -gpu=cc${CMAKE_CUDA_ARCHITECTURES}) - endif() - endif() - - if(${MPI_FOUND}) - set(MPI TRUE) - set(HDF5_MPI ${HDF5_FOUND}) - if(ops_cuda_FOUND) - set(CUDA_MPI ${CUDAToolkit_FOUND}) - if(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(OMPOFFLOAD_MPI TRUE) - endif() - endif() - - endif() - -# Prepare ctest - if(OPS_TEST) - enable_testing() - include(ProcessorCount) - ProcessorCount(CPU_NUMBER) - if(CPU_NUMBER GREATER 12) - set(CPU_NUMBER 12) - endif() - if(NOT GPU_NUMBER) - message( - WARNING "GPU tests are disabled! Please use -DGPU_NUMBER to specify!") - endif() - endif() - endif() - -# If APPs are compiled together with the OPS API - if(${CMAKE_PROJECT_NAME} STREQUAL OPS) - - include_directories("${CMAKE_SOURCE_DIR}/ops/c/include") - set(OPS_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/ops/c/include") - - set(HDF5_SEQ ${HDF5_FOUND}) - set(CUDA ${CUDAToolkit_FOUND}) - if(CUDA) - if(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(OMPOFFLOAD TRUE) - set(OMPOFFLOAD_FLAGS -Minline=reshape -Mvect=levels:5 -mp=gpu -gpu=cc${CMAKE_CUDA_ARCHITECTURES}) - endif() - endif() - - if(OPS_HIP) - set(HIP ${HIP_FOUND}) - endif() - - if(${MPI_FOUND}) - set(MPI TRUE) - set(HDF5_MPI ${HDF5_FOUND}) - set(CUDA_MPI ${CUDAToolkit_FOUND}) - if(CUDA_MPI) - if(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(OMPOFFLOAD_MPI TRUE) - endif() - endif() - endif() - - if(NOT Python3_FOUND) - message( - WARNING - "We cannot find Python3 and the OPS code-generator needs Python3! Please use -DPython3_EXECUTABLE to specify the path." - ) - else() - if(LEGACY_CODEGEN) - set(OPS_C_TRANSLATOR "${CMAKE_INSTALL_PREFIX}/translator/ops_translator_legacy/c/ops.py") - set(OPS_F_TRANSLATOR "${CMAKE_INSTALL_PREFIX}/translator/ops_translator_legacy/fortran/ops_fortran.py") - else() - set(OPS_C_TRANSLATOR "${CMAKE_INSTALL_PREFIX}/translator/ops_translator/ops-translator") - set(OPS_F_TRANSLATOR "${CMAKE_INSTALL_PREFIX}/translator/ops_translator/ops-translator") - endif() - endif() - -# Set the OPS_INSTALL_PATH for testing - set(OPS_CMAKE_INSTALL_PATH "${CMAKE_INSTALL_PREFIX}") - set(OPS_INSTALL_PATH "${CMAKE_SOURCE_DIR}/ops/c") - set(OPS_APP_SRC "${CMAKE_SOURCE_DIR}/apps/c") - - if(OPS_TEST) - enable_testing() - endif() - endif() - -# Find the MPI INCLUDE directory, required to resolve mpi.h not found error for some targets - if(${MPI_FOUND}) - if(DEFINED ENV{I_MPI_ROOT}) #Intel MPI - set(MPI_INC_LIST -I$ENV{I_MPI_ROOT}/include) - elseif(DEFINED ENV{MPI_ROOT}) # HPE MPT - set(MPI_INC_LIST -I$ENV{MPI_ROOT}/include) - elseif(DEFINED ENV{MPIROOT}) # SOME GNU SYSTEMS - set(MPI_INC_LIST -I$ENV{MPIROOT}/include) - elseif(DEFINED ENV{MPICH_DIR}) # Cray - set(MPI_INC_LIST -I$ENV{MPICH_DIR}/include) - else() # OpenMPI - execute_process(COMMAND mpicc --showme:compile - OUTPUT_VARIABLE MPI_INC_STRING - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_INC_FIND) - if(MPI_INC_FIND) - message(FATAL_ERROR "Failed to determine MPI include directories: ${MPI_INC_FIND}") - endif() - string(REPLACE " " ";" MPI_INC_LIST ${MPI_INC_STRING}) - - endif() - endif() - - -# Clean the tmp directory for the code generation - macro(CreateTempDir) - file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/tmp") - file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tmp") - endmacro() - - function(add_cmake_test EXE_NAME SCRIPT_PATH ARGUMENTS) - if(EXISTS ${SCRIPT_PATH}) - separate_arguments(args NATIVE_COMMAND ${ARGUMENTS}) - execute_process(COMMAND /bin/bash ${SCRIPT_PATH} ${EXE_NAME} ${args} - RESULT_VARIABLE test_result - OUTPUT_VARIABLE test_output) - - if(test_result EQUAL 0) - string(REGEX REPLACE "\n" "" test_output "${test_output}") - -# message(STATUS "run command: ${test_output}") - set(args " ") - set(cmd "${test_output}") - add_test(NAME ${EXE_NAME} - COMMAND ${CMAKE_COMMAND} - -DCMD=${cmd} -DARG=${args} - -DOPS_INSTALL_PATH=${OPS_CMAKE_INSTALL_PATH} -P ${OPS_APP_SRC}/runtests.cmake - WORKING_DIRECTORY "${TMP_SOURCE_DIR}") - else() - message(FATAL_ERROR "Error in executing script to get runtime command string ${test_result}: ${test_output}") - endif() - endif() - endfunction() - - function(add_cmake_h5diff_test EXE_NAME SCRIPT_PATH H5DIFF_TEST ARGUMENTS) - if(H5DIFF_TEST) - if(EXISTS ${SCRIPT_PATH}) - separate_arguments(args NATIVE_COMMAND ${ARGUMENTS}) - execute_process(COMMAND /bin/bash ${SCRIPT_PATH} ${CMAKE_CURRENT_BINARY_DIR} ${args} - RESULT_VARIABLE test_result - OUTPUT_VARIABLE test_output) - - if(test_result EQUAL 0) - string(REGEX REPLACE "\n" "" test_output "${test_output}") - -# message(STATUS "h5diff command: ${test_output}") - set(args " ") - - set(output_file "${EXE_NAME}_h5diff.sh") -# Write the content of test_output to the file - file(WRITE ${output_file} "${test_output}") - - set(cmd "/bin/bash ${output_file}") - add_test(NAME ${EXE_NAME}_h5diff - COMMAND ${CMAKE_COMMAND} - -DCMD=${cmd} -DARG=${args} - -DOPS_INSTALL_PATH=${OPS_CMAKE_INSTALL_PATH} -P ${OPS_APP_SRC}/runtests.cmake - WORKING_DIRECTORY "${TMP_SOURCE_DIR}") - else() - message(FATAL_ERROR "Error in executing script to get runtime command string ${test_result}: ${test_output}") - endif() - endif() - endif() - endfunction() - - -# Prepare the macro for compiling apps -# Name: App name -# Odd: Key words for source files are included in other source files and need no explicit compilation -# Others: Key words for source codes need no code generation using ops.py -# Extra: Key words for ource codes that cannot be treated by the macro -# Trid: Whether the tridiagonal library is required for this application -# GenerateTest: if generating the testings, some apps need specific testing targets. -# TODO: sycl, hip, ompoffload - macro( - BUILD_OPS_C_SAMPLE - Name - Odd - Others - Extra - Trid - GenerateTest) - -# Copy all source and head files into tmp - file(GLOB CPP "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") - - list(FILTER CPP EXCLUDE REGEX "ops.cpp") - - file(COPY ${CPP} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tmp/") - file(GLOB HEADS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") - file(COPY ${HEADS} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tmp/") - - set(TMP_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/tmp") - -# Skipping sequential run tests for Cloverleaf applications - string(FIND ${Name} "clover_leaf" clover_flag) - string(FIND ${Name} "mb_shsgc" mb_shsgc_flag) - if((clover_flag EQUAL -1) AND (mb_shsgc_flag EQUAL -1)) - set(SKIP_SEQ_TESTING FALSE) - else() - set(SKIP_SEQ_TESTING TRUE) - endif() - - if("${Name}" STREQUAL "lowdim" ) #OR "${Name}" STREQUAL "multidim") - set(H5DIFF_TEST TRUE) - else() - set(H5DIFF_TEST FALSE) - endif() - -# Get the file to be compiled, DEV: non-optimised codes (full directory and filename) -# OPS: codes generated by ops.py (full directory and filename) -# OTHERS: Source codes need no code generation (full directory and filename) - file(GLOB DEV "${TMP_SOURCE_DIR}/*.cpp") - list(FILTER DEV EXCLUDE REGEX ${Odd}) - foreach(oth ${Others}) - list(FILTER DEV EXCLUDE REGEX ${oth}) - endforeach() - foreach(ext ${Extra}) - list(FILTER DEV EXCLUDE REGEX ${ext}) - endforeach() - - list(GET DEV 0 Kernel) - - get_filename_component(KernelName ${Kernel} NAME_WE) - -# If not LEGACY_CODEGEN then activate the Installed Python Virtual Environment - if(NOT LEGACY_CODEGEN) - message("Activating Python Virtual Environment for Code-Generation") - set(VENV_ACTIVATE "source ${OPS_CMAKE_INSTALL_PATH}/translator/ops_translator/ops_venv/bin/activate") - - execute_process(COMMAND /bin/bash -c ${VENV_ACTIVATE} - WORKING_DIRECTORY ${TMP_SOURCE_DIR} - RESULT_VARIABLE result - OUTPUT_VARIABLE output) - - if(result EQUAL 0) - set(PYTHON_EXECUTABLE "${OPS_CMAKE_INSTALL_PATH}/translator/ops_translator/ops_venv/bin/python3") - message(STATUS "Python Virtual Enviroment activated successfully") - else() - message("command: ${VENV_ACTIVATE}") - if(NOT "${output}" STREQUAL "") - message(FATAL_ERROR "Activate Python Virtual Environment command failed with error code ${result}: ${output}") - else() - message(FATAL_ERROR "Activate Python Virtual Environment command failed with error code ${result}") - endif() - endif() - endif() - -# Run OPS code-generation - if(LEGACY_CODEGEN) - message(STATUS "Code-gen command: ${OPS_C_TRANSLATOR} ${DEV}") - execute_process(COMMAND ${OPS_C_TRANSLATOR} ${DEV} - WORKING_DIRECTORY ${TMP_SOURCE_DIR} - RESULT_VARIABLE result - OUTPUT_VARIABLE output) - else() - if(${CMAKE_PROJECT_NAME} STREQUAL OPS) - set(PYTHON_FILE_ARGS "-DOPS_ACC_IGNORE -I ${CMAKE_SOURCE_DIR}/ops/c/include --file_paths") - set(PYTHON_OPS_INCLUDE ${CMAKE_SOURCE_DIR}/ops/c/include) - endif() - if(${CMAKE_PROJECT_NAME} STREQUAL APP) - set(PYTHON_FILE_ARGS "-DOPS_ACC_IGNORE -I ${OPS_CMAKE_INSTALL_PATH}/include --file_paths") - set(PYTHON_OPS_INCLUDE ${OPS_CMAKE_INSTALL_PATH}/include) - endif() - foreach(filepath ${DEV}) - get_filename_component(filename ${filepath} NAME) - list(APPEND FILENAMES ${filename}) - endforeach() - - message(STATUS "Code-gen command: ${PYTHON_EXECUTABLE} ${OPS_C_TRANSLATOR} ${PYTHON_FILE_ARGS} ${FILENAMES}") - execute_process(COMMAND ${PYTHON_EXECUTABLE} ${OPS_C_TRANSLATOR} - -DOPS_ACC_IGNORE - -I ${PYTHON_OPS_INCLUDE} - --file_paths ${FILENAMES} - WORKING_DIRECTORY ${TMP_SOURCE_DIR} - RESULT_VARIABLE result - OUTPUT_VARIABLE output) - endif() - - if(result EQUAL 0) - message(STATUS "OPS Python code generation runs successfully") - else() - message(FATAL_ERROR "OPS Python code generation command failed with error code ${result}: ${output}") - endif() - - file(GLOB OPS "${TMP_SOURCE_DIR}/*ops*.cpp") -# message(${OPS}) - - list(FILTER OPS EXCLUDE REGEX ${Odd}) -# message(${OPS}) - - file(GLOB OTHERS "${TMP_SOURCE_DIR}/*.cpp") - foreach(OpsFile ${OPS}) - list(REMOVE_ITEM OTHERS ${OpsFile}) - endforeach() - - foreach(NotReqFile ${DEV}) - list(REMOVE_ITEM OTHERS ${NotReqFile}) - endforeach() - - list(FILTER OTHERS EXCLUDE REGEX ${Odd}) - - foreach(ext ${Extra}) - list(FILTER OTHERS EXCLUDE REGEX ${ext}) - endforeach() - -# Copy input parameters - file(GLOB_RECURSE INPUT "${CMAKE_CURRENT_SOURCE_DIR}/*.in") - file(COPY ${INPUT} DESTINATION "${TMP_SOURCE_DIR}") - if(EXISTS "${TMP_SOURCE_DIR}/user_types.h") - install(FILES "${TMP_SOURCE_DIR}/user_types.h" - DESTINATION ${APP_INSTALL_DIR}/${Name}) - endif() - - install(FILES ${INPUT} DESTINATION ${APP_INSTALL_DIR}/${Name}) - - string(FIND ${Name} "clover_leaf_3D" clover_flag_3D) - if((clover_flag GREATER -1) AND (clover_flag_3D EQUAL -1)) - execute_process(COMMAND cp ${TMP_SOURCE_DIR}/clover_bm_short.in ${TMP_SOURCE_DIR}/clover.in) - - set(copy_command " - execute_process(COMMAND \${CMAKE_COMMAND} -E copy_if_different - ${APP_INSTALL_DIR}/${Name}/clover_bm_short.in - ${APP_INSTALL_DIR}/${Name}/clover.in) - ") - install(CODE "${copy_command}") - endif() - -# TARGET: DEV_SEQ, SEQ, OPENMP, TILED - add_executable(${Name}_dev_seq ${DEV} ${OTHERS}) - target_include_directories(${Name}_dev_seq PRIVATE ${TMP_SOURCE_DIR}) - - add_executable(${Name}_seq ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") - target_include_directories(${Name}_seq PRIVATE ${TMP_SOURCE_DIR}) - - add_executable(${Name}_openmp ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") - target_include_directories(${Name}_openmp PRIVATE ${TMP_SOURCE_DIR}) - - add_executable(${Name}_tiled ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") - target_include_directories(${Name}_tiled PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_tiled PRIVATE "-DOPS_LAZY") - - if(HDF5_SEQ) - target_link_libraries(${Name}_dev_seq ops_hdf5_seq hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - target_link_libraries(${Name}_seq ops_hdf5_seq hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - target_link_libraries(${Name}_openmp ops_hdf5_seq hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - target_link_libraries(${Name}_tiled ops_hdf5_seq hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - endif() - - if(${Trid} STREQUAL "YES") - set(TRID Yes) - else() - set(TRID No) - endif() - - if(TRID) - target_link_libraries(${Name}_dev_seq ops_trid_seq -L${LIBTRID_PATH}/lib -ltridcpu) - target_link_libraries(${Name}_seq ops_trid_seq -L${LIBTRID_PATH}/lib -ltridcpu) - target_link_libraries(${Name}_openmp ops_trid_seq -L${LIBTRID_PATH}/lib -ltridcpu) - target_link_libraries(${Name}_tiled ops_trid_seq -L${LIBTRID_PATH}/lib -ltridcpu) - endif() - - target_link_libraries(${Name}_dev_seq ops_seq OpenMP::OpenMP_CXX) - target_link_libraries(${Name}_seq ops_seq OpenMP::OpenMP_CXX) - target_link_libraries(${Name}_openmp ops_seq OpenMP::OpenMP_CXX) - target_link_libraries(${Name}_tiled ops_seq OpenMP::OpenMP_CXX) - - install(TARGETS ${Name}_dev_seq DESTINATION ${APP_INSTALL_DIR}/${Name}) - install(TARGETS ${Name}_seq DESTINATION ${APP_INSTALL_DIR}/${Name}) - install(TARGETS ${Name}_openmp DESTINATION ${APP_INSTALL_DIR}/${Name}) - install(TARGETS ${Name}_tiled DESTINATION ${APP_INSTALL_DIR}/${Name}) - -# Sequential tests disabled since it is too time consuming for some tests at this moment - if ((OPS_TEST) AND (${GenerateTest} STREQUAL "YES") AND (NOT SKIP_SEQ_TESTING)) - - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_dev_seq ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh dev_seq) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_dev_seq ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_dev_seq) - - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_seq ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh seq) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_seq ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_seq) - - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_openmp ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh openmp) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_openmp ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_openmp) - - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_tiled ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh tiled) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_tiled ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_tiled) - endif() - - if(CUDA) - add_executable(${Name}_cuda ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/cuda/cuda_kernels.cu") - target_include_directories(${Name}_cuda PRIVATE ${TMP_SOURCE_DIR}) - target_compile_options(${Name}_cuda PRIVATE $<$:--fmad=false>) - target_link_libraries(${Name}_cuda ops_cuda CUDA::cudart_static -lcurand -lnvidia-ml OpenMP::OpenMP_CXX) - if(HDF5_SEQ) - target_link_libraries(${Name}_cuda ops_hdf5_seq hdf5::hdf5 hdf5::hdf5_hl) - endif() - - if(TRID) - target_link_libraries(${Name}_cuda ops_trid_cuda -L${LIBTRID_PATH}/lib -ltridcuda - CUDA::cublas_static CUDA::cublasLt_static CUDA::culibos) - endif() - - install(TARGETS ${Name}_cuda DESTINATION ${APP_INSTALL_DIR}/${Name}) - - if((OPS_TEST) - AND (GPU_NUMBER GREATER_EQUAL 1) - AND (${GenerateTest} STREQUAL "YES")) - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_cuda ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh cuda) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_cuda ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_cuda) - endif() - endif() - - if(OMPOFFLOAD) - add_executable(${Name}_ompoffload ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/openmp_offload/openmp_offload_kernels.cpp") - target_include_directories(${Name}_ompoffload PRIVATE ${TMP_SOURCE_DIR}) - target_link_libraries(${Name}_ompoffload ops_ompoffload OpenMP::OpenMP_CXX) - target_compile_options(${Name}_ompoffload PRIVATE ${OMPOFFLOAD_FLAGS}) - if(HDF5_SEQ) - target_link_libraries(${Name}_ompoffload ops_hdf5_seq hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - endif() - target_link_options(${Name}_ompoffload PRIVATE ${OMPOFFLOAD_FLAGS}) - install(TARGETS ${Name}_ompoffload DESTINATION ${APP_INSTALL_DIR}/${Name}) - - if((OPS_TEST) - AND (GPU_NUMBER GREATER_EQUAL 1) - AND (${GenerateTest} STREQUAL "YES")) - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_ompoffload ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ompoffload) - endif() - endif() - - if(OPS_HIP) - if(HIP AND NOT TRID) - add_executable(${Name}_hip ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/hip/hip_kernels.cpp") - message("${TMP_SOURCE_DIR}/hip/${KernelName}_hip_kernel.cpp") - - target_include_directories(${Name}_hip PRIVATE ${TMP_SOURCE_DIR}) - target_link_libraries(${Name}_hip ops_hip hip::device OpenMP::OpenMP_CXX) - -# if (HDF5_SEQ) target_link_libraries(${Name}_hip ops_hdf5_seq hdf5::hdf5 -# hdf5::hdf5_hl MPI::MPI_CXX) endif() - install(TARGETS ${Name}_hip DESTINATION ${APP_INSTALL_DIR}/${Name}) - if((OPS_TEST) - AND (GPU_NUMBER GREATER_EQUAL 1) - AND (${GenerateTest} STREQUAL "YES")) - - set(args "OPS_CL_DEVICE=1 OPS_BLOCK_SIZE_X=512 OPS_BLOCK_SIZE_Y=1") - add_test(NAME ${Name}_hip - COMMAND - ${CMAKE_COMMAND} -DCMD=$ -DARG=${args} - -DOPS_INSTALL_PATH=${OPS_CMAKE_INSTALL_PATH} -P - ${OPS_APP_SRC}/runtests.cmake - WORKING_DIRECTORY "${TMP_SOURCE_DIR}") - endif() - endif() - endif() - - if(MPI) - add_executable(${Name}_dev_mpi ${DEV} ${OTHERS}) - target_include_directories(${Name}_dev_mpi PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_dev_mpi PRIVATE "-DOPS_MPI") - - add_executable(${Name}_mpi ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") - target_include_directories(${Name}_mpi PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_mpi PRIVATE "-DOPS_MPI") - - add_executable(${Name}_mpi_openmp ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") - target_include_directories(${Name}_mpi_openmp PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_mpi_openmp PRIVATE "-DOPS_MPI") - - add_executable(${Name}_mpi_tiled ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") - target_include_directories(${Name}_mpi_tiled PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_mpi_tiled PRIVATE "-DOPS_MPI -DOPS_LAZY") - - if(HDF5_MPI) - target_link_libraries(${Name}_dev_mpi PRIVATE ops_hdf5_mpi hdf5::hdf5 - hdf5::hdf5_hl MPI::MPI_CXX) - target_link_libraries(${Name}_mpi PRIVATE ops_hdf5_mpi hdf5::hdf5 - hdf5::hdf5_hl MPI::MPI_CXX) - target_link_libraries(${Name}_mpi_openmp PRIVATE ops_hdf5_mpi hdf5::hdf5 - hdf5::hdf5_hl MPI::MPI_CXX) - target_link_libraries(${Name}_mpi_tiled PRIVATE ops_hdf5_mpi hdf5::hdf5 - hdf5::hdf5_hl MPI::MPI_CXX) - endif() - - if(TRID) - target_link_libraries(${Name}_dev_mpi PRIVATE ops_trid_mpi -L${LIBTRID_PATH}/lib -ltridcpu_mpi - MPI::MPI_CXX) - target_link_libraries(${Name}_mpi PRIVATE ops_trid_mpi -L${LIBTRID_PATH}/lib -ltridcpu_mpi - MPI::MPI_CXX) - target_link_libraries(${Name}_mpi_openmp PRIVATE ops_trid_mpi -L${LIBTRID_PATH}/lib -ltridcpu_mpi - MPI::MPI_CXX) - target_link_libraries(${Name}_mpi_tiled PRIVATE ops_trid_mpi -L${LIBTRID_PATH}/lib -ltridcpu_mpi - MPI::MPI_CXX) - endif() - - target_link_libraries(${Name}_dev_mpi PRIVATE ops_mpi MPI::MPI_CXX - OpenMP::OpenMP_CXX) - target_link_libraries(${Name}_mpi PRIVATE ops_mpi MPI::MPI_CXX - OpenMP::OpenMP_CXX) - target_link_libraries(${Name}_mpi_openmp PRIVATE ops_mpi MPI::MPI_CXX - OpenMP::OpenMP_CXX) - target_link_libraries(${Name}_mpi_tiled PRIVATE ops_mpi MPI::MPI_CXX - OpenMP::OpenMP_CXX) - - if((OPS_TEST) AND (${GenerateTest} STREQUAL "YES")) - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_dev_mpi ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh dev_mpi) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_dev_mpi ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_dev_mpi) - - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh mpi) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_mpi) - - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi_openmp ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh mpi_openmp) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi_openmp ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_mpi_openmp) - - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi_tiled ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh mpi_tiled) - add_cmake_h5diff_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi_tiled ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh ${H5DIFF_TEST} h5_mpi_tiled) - endif() - - install(TARGETS ${Name}_dev_mpi DESTINATION ${APP_INSTALL_DIR}/${Name}) - install(TARGETS ${Name}_mpi DESTINATION ${APP_INSTALL_DIR}/${Name}) - install(TARGETS ${Name}_mpi_openmp DESTINATION ${APP_INSTALL_DIR}/${Name}) - install(TARGETS ${Name}_mpi_tiled DESTINATION ${APP_INSTALL_DIR}/${Name}) - - if(CUDA_MPI) - add_executable(${Name}_mpi_cuda ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/cuda/cuda_kernels.cu") - target_include_directories(${Name}_mpi_cuda PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_mpi_cuda PRIVATE "-DOPS_MPI -DMPICH_IGNORE_CXX_SEEK") - target_compile_options(${Name}_mpi_cuda - PRIVATE $<$: --fmad=false >) - foreach(include_dir IN LISTS MPI_INC_LIST) - string(FIND ${include_dir} "include" include_index) - if(NOT include_index EQUAL -1) - target_compile_options(${Name}_mpi_cuda PRIVATE ${include_dir}) - endif() - endforeach() - - add_executable(${Name}_mpi_cuda_tiled ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/cuda/cuda_kernels.cu") - target_include_directories(${Name}_mpi_cuda_tiled PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_mpi_cuda_tiled PRIVATE "-DOPS_MPI -DMPICH_IGNORE_CXX_SEEK -DOPS_LAZY") - target_compile_options(${Name}_mpi_cuda_tiled - PRIVATE $<$: --fmad=false >) - foreach(include_dir IN LISTS MPI_INC_LIST) - string(FIND ${include_dir} "include" include_index) - if(NOT include_index EQUAL -1) - target_compile_options(${Name}_mpi_cuda_tiled PRIVATE ${include_dir}) - endif() - endforeach() - - if(HDF5_MPI) - target_link_libraries(${Name}_mpi_cuda PRIVATE ops_hdf5_mpi hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - target_link_libraries(${Name}_mpi_cuda_tiled PRIVATE ops_hdf5_mpi hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - endif() - - if(TRID) - target_link_libraries(${Name}_mpi_cuda PRIVATE ops_trid_mpi_cuda -L${LIBTRID_PATH}/lib -ltridcuda_mpi - MPI::MPI_CXX) - target_link_libraries(${Name}_mpi_cuda_tiled PRIVATE ops_trid_mpi_cuda -L${LIBTRID_PATH}/lib -ltridcuda_mpi - MPI::MPI_CXX) - if(NCCL_FOUND) - target_link_libraries(${Name}_mpi_cuda PRIVATE ${NCCL_LIBRARY}) - target_link_libraries(${Name}_mpi_cuda_tiled PRIVATE ${NCCL_LIBRARY}) - endif() - endif() - - target_link_libraries(${Name}_mpi_cuda PRIVATE ops_mpi_cuda CUDA::cudart_static -lcurand -lnvidia-ml MPI::MPI_CXX - OpenMP::OpenMP_CXX) - target_link_libraries(${Name}_mpi_cuda_tiled PRIVATE ops_mpi_cuda CUDA::cudart_static -lcurand -lnvidia-ml MPI::MPI_CXX - OpenMP::OpenMP_CXX) - - install(TARGETS ${Name}_mpi_cuda DESTINATION ${APP_INSTALL_DIR}/${Name}) - install(TARGETS ${Name}_mpi_cuda_tiled DESTINATION ${APP_INSTALL_DIR}/${Name}) - - if((OPS_TEST) - AND (GPU_NUMBER GREATER_EQUAL 1) - AND (${GenerateTest} STREQUAL "YES")) - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi_cuda ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh "mpi_cuda ${GPU_NUMBER}") - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi_cuda_tiled ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh "mpi_cuda_tiled ${GPU_NUMBER}") - endif() - endif() - - if(OMPOFFLOAD_MPI) - add_executable(${Name}_mpi_ompoffload ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/openmp_offload/openmp_offload_kernels.cpp") - target_include_directories(${Name}_mpi_ompoffload PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_mpi_ompoffload PRIVATE "-DOPS_MPI -DMPICH_IGNORE_CXX_SEEK") - target_compile_options(${Name}_mpi_ompoffload PRIVATE ${OMPOFFLOAD_FLAGS}) - foreach(include_dir IN LISTS MPI_INC_LIST) - string(FIND ${include_dir} "include" include_index) - if(NOT include_index EQUAL -1) - target_compile_options(${Name}_mpi_ompoffload PRIVATE ${include_dir}) - endif() - endforeach() - - add_executable(${Name}_mpi_ompoffload_tiled ${OPS} ${OTHERS} - "${TMP_SOURCE_DIR}/openmp_offload/openmp_offload_kernels.cpp") - target_include_directories(${Name}_mpi_ompoffload_tiled PRIVATE ${TMP_SOURCE_DIR}) - target_compile_definitions(${Name}_mpi_ompoffload_tiled PRIVATE "-DOPS_MPI -DMPICH_IGNORE_CXX_SEEK -DOPS_LAZY") - target_compile_options(${Name}_mpi_ompoffload_tiled PRIVATE ${OMPOFFLOAD_FLAGS}) - foreach(include_dir IN LISTS MPI_INC_LIST) - string(FIND ${include_dir} "include" include_index) - if(NOT include_index EQUAL -1) - target_compile_options(${Name}_mpi_ompoffload_tiled PRIVATE ${include_dir}) - endif() - endforeach() - - if(HDF5_MPI) - target_link_libraries(${Name}_mpi_ompoffload PRIVATE ops_hdf5_mpi hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - target_link_libraries(${Name}_mpi_ompoffload_tiled PRIVATE ops_hdf5_mpi hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - endif() - - target_link_libraries(${Name}_mpi_ompoffload PRIVATE ops_mpi_ompoffload MPI::MPI_CXX - OpenMP::OpenMP_CXX) - target_link_libraries(${Name}_mpi_ompoffload_tiled PRIVATE ops_mpi_ompoffload MPI::MPI_CXX - OpenMP::OpenMP_CXX) - - target_link_options(${Name}_mpi_ompoffload PRIVATE ${OMPOFFLOAD_FLAGS}) - target_link_options(${Name}_mpi_ompoffload_tiled PRIVATE ${OMPOFFLOAD_FLAGS}) - - install(TARGETS ${Name}_mpi_ompoffload DESTINATION ${APP_INSTALL_DIR}/${Name}) - install(TARGETS ${Name}_mpi_ompoffload_tiled DESTINATION ${APP_INSTALL_DIR}/${Name}) - - if((OPS_TEST) - AND (GPU_NUMBER GREATER_EQUAL 1) - AND (${GenerateTest} STREQUAL "YES")) - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi_ompoffload ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh "mpi_ompoffload ${GPU_NUMBER}") - add_cmake_test(${CMAKE_CURRENT_BINARY_DIR}/${Name}_mpi_ompoffload_tiled ${CMAKE_CURRENT_SOURCE_DIR}/cmake_test.sh "mpi_ompoffload_tiled ${GPU_NUMBER}") - endif() - endif() - - endif() - - endmacro() - - -# Build all listed apps - -# Current Targets: dev_seq, dev_mpi, seq, openmp, tiled, mpi, mpi_openmp, mpi_tiled -# cuda, mpi_cuda, mpi_cuda_tiled -# ompoffload, mpi_ompoffload, mpi_ompoffload_tiled (curently built only with NVHPC compilers) - -# App: Poisson # Compiles OK - add_subdirectory(poisson) - -# App: Laplace2d # Compiles OK - add_subdirectory(laplace2d_tutorial/step7) - -# App: Cloverleaf # - add_subdirectory(CloverLeaf) # Compiles OK - -# App: Cloverleaf_3D # Compiles OK, Runtime Floating point exception - add_subdirectory(CloverLeaf_3D) - -# App: Tealeaf # Compiles OK - add_subdirectory(TeaLeaf) - - if(HDF5_FOUND) -# App: Cloverleaf_3D_HDF5 # Compiles OK, Runtime Floating point exception - add_subdirectory(CloverLeaf_3D_HDF5) - -# App: lowdim # Compiles OK - add_subdirectory(lowdim_test) - -# App: multiDim # Compiles OK -# add_subdirectory(multiDim) - -# App: multiDim3D # Compiles OK -# add_subdirectory(multiDim3D) - -# App: mblock # Compiles OK -# add_subdirectory(mblock) - -# App: mgrid # Compiles OK -# add_subdirectory(mgrid) - -# App: shsgc # Compiles OK - add_subdirectory(shsgc) - -# App: hdf5_slice # Compiles OK -# add_subdirectory(hdf5_slice) -# add_subdirectory(hdf5_slice) - endif() - -# App: mb_shsgc # Compiles OK - add_subdirectory(mb_shsgc) - -# App: ops-lbm # Compiles OK - add_subdirectory(ops-lbm/step5) - -# TODO there seem bugs in the codes of complex_numbers -# add_subdirectory(complex_numbers) TODO Need to figure out the structure -# TODO there seems problem for compiling mblock4D add_subdirectory(mblock4D) # -# ignored at this moment TODO needs to replace acceptable with PASSED for -# standard test -# add_subdirectory(multiDim_HDF5) -# add_subdirectory(compact_scheme) - -# TODO fetch_data.cpp cannot be found for access add_subdirectory(access) - if(LIBTRID_PATH) - if(HDF5_FOUND) - add_subdirectory(adi) - add_subdirectory(adi_burger) - add_subdirectory(adi_burger_3D) - endif() +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +include(OPS_BUILD_APPS) + +# Create a folder directory for testing +set(app_dir_c "${app_dir}/c") +file(MAKE_DIRECTORY ${app_dir_c}) + +# Include include directory for C++ +include_directories("${CMAKE_SOURCE_DIR}/ops/c/include") + +# Set the OPS Translator +if(LEGACY_CODEGEN) + set(OPS_C_TRANSLATOR "${opsc}/ops.py") +else() + set(OPS_C_TRANSLATOR "${opsc}") +endif() + + +if(${MPI_FOUND}) + if(DEFINED ENV{I_MPI_ROOT}) #Intel MPI + set(MPI_INC_LIST -I$ENV{I_MPI_ROOT}/include) + elseif(DEFINED ENV{MPI_ROOT}) # HPE MPT + set(MPI_INC_LIST -I$ENV{MPI_ROOT}/include) + elseif(DEFINED ENV{MPICH_DIR}) # Cray + set(MPI_INC_LIST -I$ENV{MPICH_DIR}/include) + else() # OpenMPI + execute_process(COMMAND mpicc --showme:compile + OUTPUT_VARIABLE MPI_INC_STRING + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_VARIABLE MPI_INC_FIND) + if(MPI_INC_FIND) + message(FATAL_ERROR "Failed to determine MPI include directories: ${MPI_INC_FIND}") endif() + string(REPLACE " " ";" MPI_INC_LIST ${MPI_INC_STRING}) + endif() +endif() + +#add_subdirectory(access) #This does not compile +add_subdirectory(CloverLeaf) +#add_subdirectory(complex_numbers) #This does not compile +add_subdirectory(laplace2d_tutorial/step7) +add_subdirectory(ops-lbm/step5) +#add_subdirectory(lowdim_test) # Issue with mpi-cuda with multiple definition +#add_subdirectory(mblock) # Issue with mpi-cuda with multiple definition +#add_subdirectory(mgrid) # Issue with mpi-cuda with multiple definition +add_subdirectory(multiDim) +add_subdirectory(multiDim3D) +add_subdirectory(poisson) +add_subdirectory(shsgc) +if(${OPS_TEST_FULL}) + add_subdirectory(CloverLeaf_3D) + add_subdirectory(CloverLeaf_3D_HDF5) + add_subdirectory(mb_shsgc) +endif() diff --git a/apps/c/CloverLeaf/CMakeLists.txt b/apps/c/CloverLeaf/CMakeLists.txt index 86b6c5d774..141cabdb70 100644 --- a/apps/c/CloverLeaf/CMakeLists.txt +++ b/apps/c/CloverLeaf/CMakeLists.txt @@ -1,5 +1,13 @@ -cmake_minimum_required(VERSION 3.18) -set(others start.cpp time_step.cpp build_field.cpp read_input.cpp - initialise.cpp advection.cpp) +set(case "CloverLeaf") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -build_ops_c_sample(clover_leaf profile "${others}" "NONE" "NO" "YES") +#Set additional sources +set(others start.cpp + time_step.cpp + build_field.cpp + read_input.cpp + initialise.cpp + advection.cpp) +# Off files to avoid: generally they are idetified with a small string +set(odd "profile") +build_ops_c_sample(${case} ${odd} "${others}" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/CloverLeaf/cmake_test.sh b/apps/c/CloverLeaf/cmake_test.sh index 67c1408710..965034e51f 100644 --- a/apps/c/CloverLeaf/cmake_test.sh +++ b/apps/c/CloverLeaf/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 2 --oversubscribe $exe OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np 2 $exe OPS_TILING OPS_TILING_MAXDEPTH=6" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" fi diff --git a/apps/c/CloverLeaf_3D/CMakeLists.txt b/apps/c/CloverLeaf_3D/CMakeLists.txt index 98e4dc4b53..ab79906f34 100644 --- a/apps/c/CloverLeaf_3D/CMakeLists.txt +++ b/apps/c/CloverLeaf_3D/CMakeLists.txt @@ -1,5 +1,13 @@ -cmake_minimum_required(VERSION 3.18) -set(others start.cpp time_step.cpp build_field.cpp read_input.cpp - initialise.cpp advection.cpp) +set(case "CloverLeaf_3D") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -build_ops_c_sample(clover_leaf_3D profile "${others}" "NONE" "NO" "YES") +#Set additional sources +set(others start.cpp + time_step.cpp + build_field.cpp + read_input.cpp + initialise.cpp + advection.cpp) +# Off files to avoid: generally they are idetified with a small string +set(odd "profile") +build_ops_c_sample(${case} ${odd} "${others}" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/CloverLeaf_3D/cmake_test.sh b/apps/c/CloverLeaf_3D/cmake_test.sh index 56c21b3ac7..b94e485778 100644 --- a/apps/c/CloverLeaf_3D/cmake_test.sh +++ b/apps/c/CloverLeaf_3D/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 4 --oversubscribe $exe OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np 4 $exe OPS_TILING OPS_TILING_MAXDEPTH=6" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" fi diff --git a/apps/c/CloverLeaf_3D_HDF5/cmake_test.sh b/apps/c/CloverLeaf_3D_HDF5/cmake_test.sh index 56c21b3ac7..b94e485778 100644 --- a/apps/c/CloverLeaf_3D_HDF5/cmake_test.sh +++ b/apps/c/CloverLeaf_3D_HDF5/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 4 --oversubscribe $exe OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np 4 $exe OPS_TILING OPS_TILING_MAXDEPTH=6" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=6" fi diff --git a/apps/c/TeaLeaf/cmake_test.sh b/apps/c/TeaLeaf/cmake_test.sh index 4b7a060871..170778a07a 100644 --- a/apps/c/TeaLeaf/cmake_test.sh +++ b/apps/c/TeaLeaf/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 6 --oversubscribe $exe" + echo "mpirun -np 6 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 6 --oversubscribe $exe" + echo "mpirun -np 6 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 6 --oversubscribe $exe OPS_TILING" + echo "mpirun -np 6 $exe OPS_TILING" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" fi diff --git a/apps/c/access/CMakeLists.txt b/apps/c/access/CMakeLists.txt index dfe4997707..e91d32203a 100644 --- a/apps/c/access/CMakeLists.txt +++ b/apps/c/access/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) - -BUILD_OPS_C_SAMPLE(access "NONE" "NONE" "NO" "NONE") +set(case "access") +set(CASE_DIR "${app_dir_c}/${case}") +createtempdir() +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "OPS_TEST_DEV" "2") diff --git a/apps/c/cmake/OPS_BUILD_APPS.cmake b/apps/c/cmake/OPS_BUILD_APPS.cmake new file mode 100644 index 0000000000..85f8a9b630 --- /dev/null +++ b/apps/c/cmake/OPS_BUILD_APPS.cmake @@ -0,0 +1,471 @@ +# Clean the tmp directory for the code generation +macro(CreateTempDir) + file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/tmp") + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tmp") +endmacro() + +function(add_cmake_test WORK_DIR EXE_NAME SCRIPT_PATH ARGUMENTS OMPNT) + if(EXISTS ${SCRIPT_PATH}) + set(exe_file "${WORK_DIR}/${EXE_NAME}") + list(LENGTH ARGUMENTS NL) + if(NL LESS 2) + separate_arguments(args NATIVE_COMMAND ${ARGUMENTS}) + else() + message(WARNING "ARGUMENTS to indentify execute command is a list and not a string: this might not work") + endif() + execute_process(COMMAND /bin/bash ${SCRIPT_PATH} ${exe_file} ${args} + RESULT_VARIABLE test_result + OUTPUT_VARIABLE test_output) + if(test_result EQUAL 0) + string(REGEX REPLACE "\n" "" test_output "${test_output}") + #message(STATUS "run command: ${test_output}") + set(args " ") + set(cmd "${test_output}") + add_test(NAME ${EXE_NAME} + COMMAND ${CMAKE_COMMAND} + -DCMD=${cmd} -DARG=${args} -DOMPNT=${OMPNT} + -DOPS_INSTALL_PATH=${CMAKE_INSTALL_PATH} -P ${CMAKE_SOURCE_DIR}/apps/c/cmake/OPS_runtests.cmake + WORKING_DIRECTORY "${WORK_DIR}") + else() + message(FATAL_ERROR "Error in executing script to get runtime command string ${test_result}: ${test_output}") + endif() + endif() +endfunction() + + +macro(SetAppExe + APP_SRC + Root_Name + APP_type + APP_DIR_ROOT + APP_DIR_SRC + CWD + INP + Links + Defs + Options + Props + OMPNT) + # Init the lists for FLAGS and LINKS + set(Defs_Loc "") + set(Links_Loc "") + set(Opts_Loc "") + set(Props_Loc "") + list(APPEND Defs_Loc ${Defs}) + list(APPEND Links_Loc ${Links}) + list(APPEND Opts_Loc ${Options}) + list(APPEND Props_Loc ${Props}) + # Set APP names and paths + set(APP_exe ${Root_Name}_${APP_TYPE}) + set(APP_DIR_DST ${APP_DIR_ROOT}/${APP_exe}) + string(FIND ${APP_type} "mpi" FLAG_FOUND) + if(FLAG_FOUND GREATER -1) + list(APPEND Defs_Loc "-DOPS_MPI") + endif() + # + if (HDF5_FOUND) + if(FLAG_FOUND GREATER -1) + list(APPEND Links_Loc "ops_hdf5_mpi") + else() + list(APPEND Links_Loc "ops_hdf5_seq") + endif() + list(APPEND Links_Loc "hdf5::hdf5") + list(APPEND Links_Loc "hdf5::hdf5_hl") + list(APPEND Links_Loc "MPI::MPI_CXX") + endif() + # + string(FIND ${APP_type} "tiled" FLAG_FOUND) + if(FLAG_FOUND GREATER -1) + list(APPEND Defs_Loc "-DOPS_LAZY") + endif() + add_executable(${APP_exe} ${APP_SRC}) + #message(STATUS "${APP_type} APP_DIR_SRC ${APP_DIR_SRC}") + target_include_directories(${APP_exe} PRIVATE ${APP_DIR_SRC}) + foreach(Def IN LISTS Defs_Loc) + target_compile_definitions(${APP_exe} PUBLIC ${Def}) + endforeach() + foreach(Opt IN LISTS Opts_Loc) + target_compile_options(${APP_exe} PUBLIC ${Opt}) + endforeach() + list(LENGTH Props_Loc Props_length) + # These link properties needs to be passed and a single string + # Passing them individually does not work + if(Props_length GREATER 0) + string(REPLACE ";" " " Props_string "${Props_Loc}") + set_target_properties(${APP_exe} PROPERTIES LINK_FLAGS "${Props_string}") + endif() + string(FIND "${INP}" "/" position REVERSE) + if(position GREATER -1) + install(FILES ${INP} DESTINATION ${APP_DIR_DST} RENAME clover.in) + endif() + foreach(Link IN LISTS Links_Loc) + target_link_libraries(${APP_exe} PRIVATE ${Link}) + endforeach() + install(TARGETS ${APP_exe} DESTINATION ${APP_DIR_DST}) + # Append the number of GPUs + string(FIND ${APP_type} "mpi_cuda" FLAG_FOUND1) + string(FIND ${APP_type} "mpi_ompoffload" FLAG_FOUND2) + if(FLAG_FOUND1 GREATER -1 OR FLAG_FOUND2 GREATER -1) + set(ARGUMENTS "${APP_type} ${GPU_NUMBER}") + else() + set(ARGUMENTS "${APP_type}" ) + endif() + add_cmake_test(${APP_DIR_DST} ${APP_exe} ${CWD}/cmake_test.sh ${ARGUMENTS} ${OMPNT}) +endmacro() + +# Prepare the macro for compiling apps Name: App name Odd: Key words for source +# files are included in other source files and need no explicit compilation +# Others: Key words for source codes need no code generation using ops.py Extra: +# Key words for ource codes that cannot be treated by the macro Trid: Whether +# the tridiagonal library is required for this application GenerateTest: if +# generating the testings, some apps need specific testing targets. +# TODO: tiled, sycl, openmpoffload +macro( + BUILD_OPS_C_SAMPLE + Name + Odd + Others + Extra + Trid + TestDev + OMPNT) + + message(STATUS "------Build SAMPLE-----------") + + # Copy all source and head files into tmp + file(GLOB CPP "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") + + list(FILTER CPP EXCLUDE REGEX "ops.cpp") + + file(COPY ${CPP} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tmp/") + file(GLOB HEADS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") + file(COPY ${HEADS} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/tmp/") + + set(TMP_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/tmp") + + # Get the file to be compiled, DEV: non-optimised codes (full directory and filename) + # OPS: codes generated by ops.py (full directory and filename) + # OTHERS: Source codes need no code generation (full directory and filename) + file(GLOB DEV "${TMP_SOURCE_DIR}/*.cpp") + list(FILTER DEV EXCLUDE REGEX ${Odd}) + foreach(oth ${Others}) + list(FILTER DEV EXCLUDE REGEX ${oth}) + endforeach() + foreach(ext ${Extra}) + list(FILTER DEV EXCLUDE REGEX ${ext}) + endforeach() + + list(GET DEV 0 Kernel) + + get_filename_component(KernelName ${Kernel} NAME_WE) + + # If not LEGACY_CODEGEN then activate the Installed Python Virtual Environment + if(NOT LEGACY_CODEGEN) + set(VENV_ACTIVATE "source ${CMAKE_INSTALL_PREFIX}/translator/ops_translator/ops_venv/bin/activate") + + execute_process(COMMAND /bin/bash -c ${VENV_ACTIVATE} + WORKING_DIRECTORY ${TMP_SOURCE_DIR} + RESULT_VARIABLE result + OUTPUT_VARIABLE output) + + if(result EQUAL 0) + set(PYTHON_EXECUTABLE "${CMAKE_INSTALL_PREFIX}/translator/ops_translator/ops_venv/bin/python3") + message(STATUS "Python Virtual Enviroment activated successfully") + else() + message("command: ${VENV_ACTIVATE}") + if(NOT "${output}" STREQUAL "") + message(FATAL_ERROR "Activate Python Virtual Environment command failed with error code ${result}: ${output}") + else() + message(FATAL_ERROR "Activate Python Virtual Environment command failed with error code ${result}") + endif() + endif() + endif() + + # Run OPS code-generation + message(STATUS "Start Code Generation") + if(LEGACY_CODEGEN) + message(STATUS "Code-gen command legacy: ${OPS_C_TRANSLATOR} ${DEV}") + execute_process(COMMAND ${OPS_C_TRANSLATOR} ${DEV} + WORKING_DIRECTORY ${TMP_SOURCE_DIR} + RESULT_VARIABLE result + OUTPUT_VARIABLE output) + else() + if(${CMAKE_PROJECT_NAME} STREQUAL OPS) + set(PYTHON_FILE_ARGS "-DOPS_ACC_IGNORE -I ${CMAKE_SOURCE_DIR}/ops/c/include --file_paths") + set(PYTHON_OPS_INCLUDE ${CMAKE_SOURCE_DIR}/ops/c/include) + endif() + if(${CMAKE_PROJECT_NAME} STREQUAL APP) + set(PYTHON_FILE_ARGS "-DOPS_ACC_IGNORE -I ${CMAKE_INSTALL_PATH}/include --file_paths") + set(PYTHON_OPS_INCLUDE ${CMAKE_INSTALL_PATH}/include) + endif() + foreach(filepath ${DEV}) + get_filename_component(filename ${filepath} NAME) + list(APPEND FILENAMES ${filename}) + endforeach() + + message(STATUS "Code-gen command: ${PYTHON_EXECUTABLE} ${OPS_C_TRANSLATOR} ${PYTHON_FILE_ARGS} ${FILENAMES}") + execute_process(COMMAND ${PYTHON_EXECUTABLE} ${OPS_C_TRANSLATOR} + -DOPS_ACC_IGNORE + -I ${PYTHON_OPS_INCLUDE} + --file_paths ${FILENAMES} + WORKING_DIRECTORY ${TMP_SOURCE_DIR} + RESULT_VARIABLE result + OUTPUT_VARIABLE output) + endif() + + if(result EQUAL 0) + message(STATUS "OPS Python code generation runs successfully") + else() + message(FATAL_ERROR "OPS Python code generation command failed with error code ${result}: ${output}") + endif() + + file(GLOB OPS "${TMP_SOURCE_DIR}/*ops*.cpp") + + list(FILTER OPS EXCLUDE REGEX ${Odd}) + + file(GLOB OTHERS "${TMP_SOURCE_DIR}/*.cpp") + foreach(OpsFile ${OPS}) + list(REMOVE_ITEM OTHERS ${OpsFile}) + string(REPLACE "_ops" "" NoOPS ${OpsFile}) + list(REMOVE_ITEM OTHERS ${NoOPS}) + endforeach() + + list(FILTER OTHERS EXCLUDE REGEX ${Odd}) + + foreach(ext ${Extra}) + list(FILTER OTHERS EXCLUDE REGEX ${ext}) + endforeach() + + # Copy input parameters + file(GLOB_RECURSE INP "${CMAKE_CURRENT_SOURCE_DIR}/*bm_short.in") + + # TARGET: DEV_SEQ, SEQ, OPENMP, TILED + set(Defs "") + set(Opts "") + set(Props "") + set(APP_SRC ${DEV} ${OTHERS}) + if(${TestDev}) + message(STATUS "Set UP also DEV") + set(APP_TYPE "dev_seq") + if (USE_OMP) + set(Links "ops_seq" + "OpenMP::OpenMP_CXX") + else () + set(Links "ops_seq") + endif() + # Make sure to use "" for potentially empty inputs + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + if(MPI_FOUND) + set(APP_TYPE "dev_mpi") + if (USE_OMP) + set(Links "ops_mpi" + "OpenMP::OpenMP_CXX") + else () + set(Links "ops_mpi") + endif() + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + endif() + endif() + + set(APP_TYPE "seq") + set(APP_SRC ${OPS} ${OTHERS} "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") + if (USE_OMP) + set(Links "ops_seq" + "OpenMP::OpenMP_CXX") + else () + set(Links "ops_seq") + endif() + set(Defs "") + set(Opts "") + set(Props "") + # Make sure to use "" for potentially empty inputs + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + if(MPI_FOUND) + set(APP_TYPE "mpi") + if (USE_OMP) + set(Links "ops_mpi" + "OpenMP::OpenMP_CXX") + else () + set(Links "ops_mpi") + endif() + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + endif() + + if (USE_OMP) + set(APP_TYPE "openmp") + set(APP_SRC ${OPS} ${OTHERS} + "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") + set(Links "ops_seq" + "OpenMP::OpenMP_CXX") + set(Defs "") + set(Opts "") + set(Props "") + # Make sure to use "" for potentially empty inputs + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + if(MPI_FOUND) + set(APP_TYPE "mpi_openmp") + set(Links "ops_mpi" + "OpenMP::OpenMP_CXX") + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + endif() + endif() + + set(APP_TYPE "tiled") + set(APP_SRC ${OPS} ${OTHERS} + "${TMP_SOURCE_DIR}/mpi_openmp/mpi_openmp_kernels.cpp") + if (USE_OMP) + set(Links "ops_seq" + "OpenMP::OpenMP_CXX") + else () + set(Links "ops_seq") + endif() + set(Defs "") + set(Opts "") + set(Props "") + # Make sure to use "" for potentially empty inputs + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + if(MPI_FOUND) + set(APP_TYPE "mpi_tiled") + if (USE_OMP) + set(Links "ops_mpi" + "OpenMP::OpenMP_CXX") + else () + set(Links "ops_mpi") + endif() + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + endif() + + if(CUDAToolkit_FOUND) + set(APP_TYPE "cuda") + set(APP_SRC ${OPS} ${OTHERS} + "${TMP_SOURCE_DIR}/cuda/cuda_kernels.cu") + if (USE_OMP) + set(Links "ops_cuda" + "CUDA::cudart_static" + "-lcurand" + "OpenMP::OpenMP_CXX") + else () + set(Links "ops_cuda" + "CUDA::cudart_static" + "-lcurand") + endif() + + set(Defs "") + # Not sure this should be here: why not on the general nvcc flags? + set(Opts "") + set(Props "") + # Make sure to use "" for potentially empty inputs + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + if(MPI_FOUND) + set(APP_TYPE "mpi_cuda") + set(Defs "-DMPICH_IGNORE_CXX_SEEK") + if (USE_OMP) + set(Links "ops_mpi_cuda" + "CUDA::cudart_static" + "-lcurand" + "OpenMP::OpenMP_CXX") + else() + set(Links "ops_mpi_cuda" + "CUDA::cudart_static" + "-lcurand") + endif() + list(APPEND Opts ${MPI_INC_LIST}) + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + # + set(APP_TYPE "mpi_cuda_tiled") + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + endif() + endif() + # HIP + if(HIP_FOUND) + set(APP_TYPE "hip") + set(APP_SRC ${OPS} ${OTHERS} + "${TMP_SOURCE_DIR}/hip/hip_kernels.cpp") + set(Links "ops_hip" + "hip::device" + "OpenMP::OpenMP_CXX") + set(Defs "-D__HIP_PLATFORM_NVIDIA__") + # Not sure this should be here: why not on the general nvcc flags? + set(Opts "") + set(Props "") + # Make sure to use "" for potentially empty inputs + message(STATUS "HIP SRC ${APP_SRC} for ${APP_TYPE}") + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + endif() + # OMPOFFLOAD + if(OPS_CXXFLAGS_OMPOFFLOAD) + set(APP_TYPE "ompoffload") + set(APP_SRC ${OPS} ${OTHERS} + "${TMP_SOURCE_DIR}/openmp_offload/openmp_offload_kernels.cpp") + set(Links "ops_ompoffload" + "OpenMP::OpenMP_CXX") + set(Defs "") + set(Opts "") + set(Props "") + foreach(Flag IN LISTS OPS_CXXFLAGS_OMPOFFLOAD) + set(Opt "$<$:${Flag}>") + list(APPEND Opts "${Opt}") + list(APPEND Props "${Flag}") + endforeach() + # Make sure to use "" for potentially empty inputs + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + if(MPI_FOUND) + set(APP_TYPE "mpi_ompoffload") + set(Defs "-DMPICH_IGNORE_CXX_SEEK") + set(Links "ops_mpi_ompoffload" + "OpenMP::OpenMP_CXX") + list(APPEND Opts ${MPI_INC_LIST}) + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + # + set(APP_TYPE "mpi_ompoffload_tiled") + setappexe("${APP_SRC}" "${Name}" "${APP_TYPE}" + "${app_dir_c}" "${TMP_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" "${INP}" + "${Links}" "${Defs}" "${Opts}" "${Props}" "${OMPNT}") + endif() + + endif() + +endmacro() diff --git a/apps/c/runtests.cmake b/apps/c/cmake/OPS_runtests.cmake similarity index 67% rename from apps/c/runtests.cmake rename to apps/c/cmake/OPS_runtests.cmake index 044474499f..526ba06d5c 100644 --- a/apps/c/runtests.cmake +++ b/apps/c/cmake/OPS_runtests.cmake @@ -1,23 +1,21 @@ -macro(TEST CMD ARG OPS_INSTALL_PATH) - set(ENV{OPS_INSTALL_PATH} ${OPS_INSTALL_PATH}) - +macro(TEST CMD ARG OMPNT) string(FIND ${CMD} "_openmp" openmp_index) if(openmp_index EQUAL -1) set(ENV{OMP_NUM_THREADS} 1) else() - set(ENV{OMP_NUM_THREADS} 2) + set(ENV{OMP_NUM_THREADS} ${OMPNT}) endif() - #set(ENV{OMP_NUM_THREADS} 1) separate_arguments(args NATIVE_COMMAND ${ARG}) separate_arguments(cmds NATIVE_COMMAND ${CMD}) + execute_process(COMMAND echo $OMP_NUM_THREADS) execute_process(COMMAND ${cmds} ${args} OUTPUT_FILE perf.out) + execute_process(COMMAND tail -n 10 perf.out) execute_process(COMMAND grep "PASSED" perf.out RESULT_VARIABLE RES) - # message("RES=${RES}") if(RES) message(FATAL_ERROR "Error running ${CMD}") endif() endmacro() -TEST(${CMD} ${ARG} ${OPS_INSTALL_PATH}) +TEST(${CMD} ${ARG} ${OMPNT}) diff --git a/apps/c/complex_numbers/CMakeLists.txt b/apps/c/complex_numbers/CMakeLists.txt index 3ec05d5249..0070fffb9b 100644 --- a/apps/c/complex_numbers/CMakeLists.txt +++ b/apps/c/complex_numbers/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) - -build_ops_c_sample(complex_numbers "NONE" "NONE" "NO" "NONE") +set(case "complex_numbers") +set(CASE_DIR "${app_dir_c}/${case}") +createtempdir() +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/laplace2d_tutorial/step7/CMakeLists.txt b/apps/c/laplace2d_tutorial/step7/CMakeLists.txt index 10a14ba32c..976c48bb6e 100644 --- a/apps/c/laplace2d_tutorial/step7/CMakeLists.txt +++ b/apps/c/laplace2d_tutorial/step7/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) +set(case "laplace2d") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -build_ops_c_sample(laplace2d "NONE" "NONE" "NONE" "NO" "YES") +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/laplace2d_tutorial/step7/cmake_test.sh b/apps/c/laplace2d_tutorial/step7/cmake_test.sh index 9cf4e44dea..e860f87cfc 100644 --- a/apps/c/laplace2d_tutorial/step7/cmake_test.sh +++ b/apps/c/laplace2d_tutorial/step7/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 6 --oversubscribe $exe" + echo "mpirun -np 6 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 6 --oversubscribe $exe" + echo "mpirun -np 6 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 6 --oversubscribe $exe" + echo "mpirun -np 6 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 6 --oversubscribe $exe OPS_TILING" + echo "mpirun -np 6 $exe OPS_TILING" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" fi diff --git a/apps/c/lowdim_test/CMakeLists.txt b/apps/c/lowdim_test/CMakeLists.txt index caa59db7a6..a6ec4fe7a2 100644 --- a/apps/c/lowdim_test/CMakeLists.txt +++ b/apps/c/lowdim_test/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) -CreateTempDir() -BUILD_OPS_C_SAMPLE(lowdim "NONE" "NONE" "NONE" "NO" "YES") +set(case "lowdim") +set(CASE_DIR "${app_dir_c}/${case}") +createtempdir() +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/mb_shsgc/CMakeLists.txt b/apps/c/mb_shsgc/CMakeLists.txt index 1c1d4397e3..475c2d5c7e 100644 --- a/apps/c/mb_shsgc/CMakeLists.txt +++ b/apps/c/mb_shsgc/CMakeLists.txt @@ -1,3 +1 @@ -cmake_minimum_required(VERSION 3.8) - add_subdirectory(Max_datatransfer) diff --git a/apps/c/mb_shsgc/Max_datatransfer/CMakeLists.txt b/apps/c/mb_shsgc/Max_datatransfer/CMakeLists.txt index a8253c6986..00c1ac1158 100644 --- a/apps/c/mb_shsgc/Max_datatransfer/CMakeLists.txt +++ b/apps/c/mb_shsgc/Max_datatransfer/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) -CreateTempDir() -BUILD_OPS_C_SAMPLE(mb_shsgc "NONE" "NONE" "NONE" "NO" "YES") +set(case "mb_shsgc") +set(CASE_DIR "${app_dir_c}/${case}") +createtempdir() +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/mb_shsgc/Max_datatransfer/cmake_test.sh b/apps/c/mb_shsgc/Max_datatransfer/cmake_test.sh index 4b7a060871..170778a07a 100644 --- a/apps/c/mb_shsgc/Max_datatransfer/cmake_test.sh +++ b/apps/c/mb_shsgc/Max_datatransfer/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 6 --oversubscribe $exe" + echo "mpirun -np 6 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 6 --oversubscribe $exe" + echo "mpirun -np 6 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 6 --oversubscribe $exe OPS_TILING" + echo "mpirun -np 6 $exe OPS_TILING" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING" fi diff --git a/apps/c/mblock/CMakeLists.txt b/apps/c/mblock/CMakeLists.txt index 7364b4881a..523c2f0097 100644 --- a/apps/c/mblock/CMakeLists.txt +++ b/apps/c/mblock/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) +set(case "mblock") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -build_ops_c_sample(mblock "NONE" "NONE" "NONE" "NO" "YES") +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${TestDev}" "2") diff --git a/apps/c/mgrid/CMakeLists.txt b/apps/c/mgrid/CMakeLists.txt index 99ccb540df..6e79fe6999 100644 --- a/apps/c/mgrid/CMakeLists.txt +++ b/apps/c/mgrid/CMakeLists.txt @@ -1,2 +1,4 @@ -cmake_minimum_required(VERSION 3.18) -build_ops_c_sample(mgrid "NONE" "NONE" "NONE" "NO" "YES") +set(case "mgrid") +set(CASE_DIR "${app_dir_c}/${case}") +createtempdir() +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}") diff --git a/apps/c/multiDim/CMakeLists.txt b/apps/c/multiDim/CMakeLists.txt index 112ed7a1aa..acd6f10b64 100644 --- a/apps/c/multiDim/CMakeLists.txt +++ b/apps/c/multiDim/CMakeLists.txt @@ -1,10 +1,4 @@ -cmake_minimum_required(VERSION 3.18) +set(case "multidim") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -if(CPU_NUMBER GREATER 4) - set(OringalCPUNum ${CPU_NUMBER}) - set(CPU_NUMBER 4) -else() - # Note that undefined variables would be `""` thus false -endif() -build_ops_c_sample(multidim "NONE" "NONE" "NONE" "NO" "YES") -set(CPU_NUMBER ${OringalCPUNum}) +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "1") diff --git a/apps/c/multiDim/cmake_test.sh b/apps/c/multiDim/cmake_test.sh new file mode 100644 index 0000000000..d25afd4245 --- /dev/null +++ b/apps/c/multiDim/cmake_test.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +exe=$1 +target=$2 +gpu_number=$3 + +if [[ "$target" == "dev_seq" ]]; then + echo "$exe" +elif [[ "$target" == "dev_mpi" ]]; then + echo "mpirun -np 4 $exe" +elif [[ "$target" == "seq" ]]; then + echo "$exe" +elif [[ "$target" == "openmp" ]]; then + echo "$exe" +elif [[ "$target" == "tiled" ]]; then + echo "$exe OPS_TILING" +elif [[ "$target" == "mpi" ]]; then + echo "mpirun -np 4 $exe" +elif [[ "$target" == "mpi_openmp" ]]; then + echo "mpirun -np 4 $exe" +elif [[ "$target" == "mpi_tiled" ]]; then + echo "mpirun -np 4 $exe OPS_TILING OPS_TILING_MAXDEPTH=10" +elif [[ "$target" == "cuda" ]]; then + echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" +elif [[ "$target" == "mpi_cuda" ]]; then + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" +elif [[ "$target" == "mpi_cuda_tiled" ]]; then + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" +elif [[ "$target" == "ompoffload" ]]; then + echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" +elif [[ "$target" == "mpi_ompoffload" ]]; then + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" +elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" +fi diff --git a/apps/c/multiDim3D/CMakeLists.txt b/apps/c/multiDim3D/CMakeLists.txt index 033a0c2ad3..8b2d8e2faf 100644 --- a/apps/c/multiDim3D/CMakeLists.txt +++ b/apps/c/multiDim3D/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) +set(case "multidim3d") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -build_ops_c_sample(multidim3d "NONE" "NONE" "NONE" "NO" "YES") +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "1") diff --git a/apps/c/multiDim3D/cmake_test.sh b/apps/c/multiDim3D/cmake_test.sh new file mode 100644 index 0000000000..d25afd4245 --- /dev/null +++ b/apps/c/multiDim3D/cmake_test.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +exe=$1 +target=$2 +gpu_number=$3 + +if [[ "$target" == "dev_seq" ]]; then + echo "$exe" +elif [[ "$target" == "dev_mpi" ]]; then + echo "mpirun -np 4 $exe" +elif [[ "$target" == "seq" ]]; then + echo "$exe" +elif [[ "$target" == "openmp" ]]; then + echo "$exe" +elif [[ "$target" == "tiled" ]]; then + echo "$exe OPS_TILING" +elif [[ "$target" == "mpi" ]]; then + echo "mpirun -np 4 $exe" +elif [[ "$target" == "mpi_openmp" ]]; then + echo "mpirun -np 4 $exe" +elif [[ "$target" == "mpi_tiled" ]]; then + echo "mpirun -np 4 $exe OPS_TILING OPS_TILING_MAXDEPTH=10" +elif [[ "$target" == "cuda" ]]; then + echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" +elif [[ "$target" == "mpi_cuda" ]]; then + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" +elif [[ "$target" == "mpi_cuda_tiled" ]]; then + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" +elif [[ "$target" == "ompoffload" ]]; then + echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" +elif [[ "$target" == "mpi_ompoffload" ]]; then + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" +elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" +fi diff --git a/apps/c/ops-lbm/step5/CMakeLists.txt b/apps/c/ops-lbm/step5/CMakeLists.txt index 68bda9e37a..b7e1c80fb5 100644 --- a/apps/c/ops-lbm/step5/CMakeLists.txt +++ b/apps/c/ops-lbm/step5/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) +set(case "lattboltz2d") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -build_ops_c_sample(lattboltz2d "NONE" "NONE" "NONE" "NO" "YES") +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/ops-lbm/step5/cmake_test.sh b/apps/c/ops-lbm/step5/cmake_test.sh index 822fd7fdc0..725ef0df09 100644 --- a/apps/c/ops-lbm/step5/cmake_test.sh +++ b/apps/c/ops-lbm/step5/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 2 --oversubscribe $exe OPS_TILING OPS_TILING_MAXDEPTH=10" + echo "mpirun -np 2 $exe OPS_TILING OPS_TILING_MAXDEPTH=10" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" fi diff --git a/apps/c/poisson/CMakeLists.txt b/apps/c/poisson/CMakeLists.txt index ae4d9f4320..7393e4de7d 100644 --- a/apps/c/poisson/CMakeLists.txt +++ b/apps/c/poisson/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) +set(case "poisson") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -build_ops_c_sample(poisson "NONE" "NONE" "NONE" "NO" "YES") +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/poisson/cmake_test.sh b/apps/c/poisson/cmake_test.sh index eef070964b..d25afd4245 100644 --- a/apps/c/poisson/cmake_test.sh +++ b/apps/c/poisson/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 4 --oversubscribe $exe" + echo "mpirun -np 4 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 4 --oversubscribe $exe OPS_TILING OPS_TILING_MAXDEPTH=10" + echo "mpirun -np 4 $exe OPS_TILING OPS_TILING_MAXDEPTH=10" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=10" fi diff --git a/apps/c/poisson/poisson.cpp b/apps/c/poisson/poisson.cpp index e181dac179..08794151a9 100644 --- a/apps/c/poisson/poisson.cpp +++ b/apps/c/poisson/poisson.cpp @@ -257,27 +257,27 @@ int main(int argc, char **argv) } } - if (non_copy) { - for (int j = 0; j < ngrid_y; j++) { - for (int i = 0; i < ngrid_x; i++) { - int iter_range[] = {0,sizes[2*(i+ngrid_x*j)],0,sizes[2*(i+ngrid_x*j)+1]}; - ops_par_loop(poisson_kernel_stencil, "poisson_kernel_stencil", blocks[i+ngrid_x*j], 2, iter_range, - ops_arg_dat(u2[i+ngrid_x*j], 1, S2D_00_P10_M10_0P1_0M1, "double", OPS_READ), - ops_arg_dat(f[i+ngrid_x*j], 1, S2D_00, "double", OPS_READ), - ops_arg_dat(u[i+ngrid_x*j], 1, S2D_00, "double", OPS_WRITE)); - } - } - } else { - for (int j = 0; j < ngrid_y; j++) { - for (int i = 0; i < ngrid_x; i++) { - int iter_range[] = {0,sizes[2*(i+ngrid_x*j)],0,sizes[2*(i+ngrid_x*j)+1]}; - ops_par_loop(poisson_kernel_update, "poisson_kernel_update", blocks[i+ngrid_x*j], 2, iter_range, - ops_arg_dat(u2[i+ngrid_x*j], 1, S2D_00, "double", OPS_READ), - ops_arg_dat(u[i+ngrid_x*j] , 1, S2D_00, "double", OPS_WRITE)); - } - } - } -// if (iter == 5) u[0] = ops_dat_copy(u[0]); //TESTING + if (non_copy) { + for (int j = 0; j < ngrid_y; j++) { + for (int i = 0; i < ngrid_x; i++) { + int iter_range[] = {0,sizes[2*(i+ngrid_x*j)],0,sizes[2*(i+ngrid_x*j)+1]}; + ops_par_loop(poisson_kernel_stencil, "poisson_kernel_stencil", blocks[i+ngrid_x*j], 2, iter_range, + ops_arg_dat(u2[i+ngrid_x*j], 1, S2D_00_P10_M10_0P1_0M1, "double", OPS_READ), + ops_arg_dat(f[i+ngrid_x*j], 1, S2D_00, "double", OPS_READ), + ops_arg_dat(u[i+ngrid_x*j], 1, S2D_00, "double", OPS_WRITE)); + } + } + } else { + for (int j = 0; j < ngrid_y; j++) { + for (int i = 0; i < ngrid_x; i++) { + int iter_range[] = {0,sizes[2*(i+ngrid_x*j)],0,sizes[2*(i+ngrid_x*j)+1]}; + ops_par_loop(poisson_kernel_update, "poisson_kernel_update", blocks[i+ngrid_x*j], 2, iter_range, + ops_arg_dat(u2[i+ngrid_x*j], 1, S2D_00, "double", OPS_READ), + ops_arg_dat(u[i+ngrid_x*j] , 1, S2D_00, "double", OPS_WRITE)); + } + } + } + //if (iter == 5) u[0] = ops_dat_copy(u[0]); //TESTING } ops_execute(blocks[0]->instance); diff --git a/apps/c/shsgc/CMakeLists.txt b/apps/c/shsgc/CMakeLists.txt index c554efc236..1f1dea23b2 100644 --- a/apps/c/shsgc/CMakeLists.txt +++ b/apps/c/shsgc/CMakeLists.txt @@ -1,3 +1,4 @@ -cmake_minimum_required(VERSION 3.18) +set(case "shsgc") +set(CASE_DIR "${app_dir_c}/${case}") createtempdir() -build_ops_c_sample(shsgc "NONE" "NONE" "NONE" "NO" "YES") +build_ops_c_sample(${case} "NONE" "NONE" "NONE" "NO" "${OPS_TEST_DEV}" "2") diff --git a/apps/c/shsgc/cmake_test.sh b/apps/c/shsgc/cmake_test.sh index 1c95865571..5b80de6c9f 100644 --- a/apps/c/shsgc/cmake_test.sh +++ b/apps/c/shsgc/cmake_test.sh @@ -7,7 +7,7 @@ gpu_number=$3 if [[ "$target" == "dev_seq" ]]; then echo "$exe" elif [[ "$target" == "dev_mpi" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "seq" ]]; then echo "$exe" elif [[ "$target" == "openmp" ]]; then @@ -15,21 +15,21 @@ elif [[ "$target" == "openmp" ]]; then elif [[ "$target" == "tiled" ]]; then echo "$exe OPS_TILING" elif [[ "$target" == "mpi" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "mpi_openmp" ]]; then - echo "mpirun -np 2 --oversubscribe $exe" + echo "mpirun -np 2 $exe" elif [[ "$target" == "mpi_tiled" ]]; then - echo "mpirun -np 2 --oversubscribe $exe OPS_TILING OPS_TILING_MAXDEPTH=8" + echo "mpirun -np 2 $exe OPS_TILING OPS_TILING_MAXDEPTH=8" elif [[ "$target" == "cuda" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_cuda_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=8" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=8" elif [[ "$target" == "ompoffload" ]]; then echo "$exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4" elif [[ "$target" == "mpi_ompoffload_tiled" ]]; then - echo "mpirun -np $gpu_number --oversubscribe $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=8" + echo "mpirun -np $gpu_number $exe OPS_BLOCK_SIZE_X=64 OPS_BLOCK_SIZE_Y=4 OPS_TILING OPS_TILING_MAXDEPTH=8" fi diff --git a/cmake/OPS_Compilers.cmake b/cmake/OPS_Compilers.cmake new file mode 100644 index 0000000000..6aae09f75b --- /dev/null +++ b/cmake/OPS_Compilers.cmake @@ -0,0 +1,62 @@ +# Compilers CMakeLists + +set(CMAKE_CXX_STANDARD 11) +set(CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER_ID} ) +message(STATUS "COMP ID CXX ${CXX_COMPILER_NAME}") +message(STATUS "CXX compiler version ${CMAKE_CXX_COMPILER_VERSION}") + +if (CXX_COMPILER_NAME MATCHES "GNU") + message(STATUS "Setting GNU flags") + include(OPS_flags_gnu) +elseif (CXX_COMPILER_NAME MATCHES "Intel") + message(STATUS "Setting Intel flags") + include(OPS_flags_intel) +elseif (CXX_COMPILER_NAME MATCHES "NVHPC") + message(STATUS "Setting NVHPC flags") + include(OPS_flags_nvhpc) +elseif (CXX_COMPILER_NAME MATCHES "Clang") + message(STATUS "Setting Clang flags") + include(OPS_flags_clang) +elseif (CXX_COMPILER_NAME MATCHES "MSVC") + message(STATUS "Setting MSVC flags") + include(OPS_flags_msvc) +else (CXX_COMPILER_NAME MATCHES "GNU") + message ("CMAKE_CXX_COMPILER full path: " ${CMAKE_CXX_COMPILER}) + message ("CXX compiler: " ${CXX_COMPILER_NAME}) + message ("No optimized CXX compiler flags are known, we just try -O2...") + set(OPS_CXXFLAGS_RELEASE "-O2") + set(OPS_CXXFLAGS_DEBUG "-O0 -g") + if(OPS_FORTRAN) + set(OPS_FORTFLAGS_RELEASE "-O2") + set(OPS_FORTFLAGS_DEBUG "-O0 -g") + endif() +endif (CXX_COMPILER_NAME MATCHES "GNU") + +if (NOT FLAGS_SET) + set(FLAGS_SET 1 CACHE INTERNAL "Flags are set") + # CXX + set(CMAKE_CXX_FLAGS ${OPS_CXXFLAGS} CACHE STRING + "Base CXXFLAGS for build" FORCE) + set(CMAKE_CXX_FLAGS_RELEASE ${OPS_CXXFLAGS_RELEASE} CACHE STRING + "Additional CXXFLAGS for Release (optimised) build" FORCE) + set(CMAKE_CXX_FLAGS_DEBUG ${OPS_CXXFLAGS_DEBUG} CACHE STRING + "Additional CXXFLAGS for Debug build" FORCE) + # If CUDA found + if(CUDAToolkit_FOUND) + set(CMAKE_CUDA_FLAGS ${OPS_CUDAFLAGS} CACHE STRING + "Base CUDAFLAGS for build" FORCE) + set(CMAKE_CUDA_FLAGS_RELEASE ${OPS_CUDAFLAGS_RELEASE} CACHE STRING + "Additional CUDAFLAGS for Release (optimised) build" FORCE) + set(CMAKE_CUDA_FLAGS_DEBUG ${OPS_CUDAFLAGS_DEBUG} CACHE STRING + "Additional CUDAFLAGS for Debug build" FORCE) + endif(CUDAToolkit_FOUND) + # FORTRAN + if(OPS_FORTRAN) + set(CMAKE_Fortran_FLAGS ${OPS_FORTFLAGS} CACHE STRING + "Base Fortran FLAGS for build" FORCE) + set(CMAKE_Fortran_FLAGS_RELEASE ${OPS_FORTFLAGS_RELEASE} CACHE STRING + "Additional Fortran FLAGS for Release (optimised) build" FORCE) + set(CMAKE_Fortran_FLAGS_DEBUG ${OPS_FORTFLAGS_DEBUG} CACHE STRING + "Additional Fortran FLAGS for Debug build" FORCE) + endif() +endif() diff --git a/cmake/OPS_FindPackages.cmake b/cmake/OPS_FindPackages.cmake new file mode 100644 index 0000000000..a67e4b8fd4 --- /dev/null +++ b/cmake/OPS_FindPackages.cmake @@ -0,0 +1,33 @@ +# Look for dependancies + +# OpenMP +include(OPS_OMP) +# HDF5 +find_package(HDF5 QUIET COMPONENTS C HL) +if(NOT HDF5_FOUND) + message(WARNING "HDF5 support NOT FOUND. The HDF5 IO routines won't work! Please use -DHDF5_ROOT to specify the path!") +else() + set(HDF5_PREFER_PARALLEL true) +endif() +# Python +find_package(Python3 REQUIRED) +if(NOT Python3_FOUND) + message(FATAL_ERROR "Python3 support NOT FOUND! The Python translator needs Python3! Please use -DPython3_EXECUTABLE to specify the path.") +endif() +# CUDA Support +# Set the NUMBER of GPU to 1 in any case +# CUDA and HIP will look if more are available +set(GPU_NUMBER 1) +if(OPS_CUDA) + include(OPS_CUDA) +endif() +# HIP Support +if(OPS_HIP) + include(OPS_HIP) +endif() +# SYCL (only for Intel) +if(CMAKE_CXX_COMPILER_ID MATCHES "Intel") + find_package(IntelSYCL QUIET) +endif() +# MPI +include(OPS_MPI) diff --git a/cmake/OPS_Fortran.cmake b/cmake/OPS_Fortran.cmake new file mode 100644 index 0000000000..549e1b3ccb --- /dev/null +++ b/cmake/OPS_Fortran.cmake @@ -0,0 +1,10 @@ +if(OPS_FORTRAN) + include(CheckLanguage) + check_language(Fortran) + if(CMAKE_Fortran_COMPILER) + enable_language(Fortran) + else() + message(WARNING "No Fortran support: fortran build is now deactivated") + set(OPS_FORTRAN OFF CACHE BOOL "Turn on the Fortran build" FORCE) + endif() +endif() diff --git a/cmake/compilers/OPS_flags_clang.cmake b/cmake/compilers/OPS_flags_clang.cmake new file mode 100644 index 0000000000..dcc45593c2 --- /dev/null +++ b/cmake/compilers/OPS_flags_clang.cmake @@ -0,0 +1,12 @@ +# Flags for Clang compiler +# CXX +set(OPS_CXXFLAGS "-fPIC -fPIC -Wall -g -std=c++11 -g") +set(OPS_CXXFLAGS_RELEASE "-O3") +set(OPS_CXXFLAGS_DEBUG "-O0") +if(OpenMP_CXX_FOUND) + set(OPS_CXXFLAGS_RELEASE "${OPS_CXXFLAGS_RELEASE} -fopenmp") +endif(OpenMP_CXX_FOUND) +if(IEEE) + set(OPS_CXXFLAGS_RELEASE "${OPS_CXXFLAGS_RELEASE} -fno-fast-math -ffp-contract=off -fdenormal-fp-math=ieee -fno-associative-math") +endif(IEEE) + diff --git a/cmake/compilers/OPS_flags_gnu.cmake b/cmake/compilers/OPS_flags_gnu.cmake new file mode 100644 index 0000000000..9519ff22b1 --- /dev/null +++ b/cmake/compilers/OPS_flags_gnu.cmake @@ -0,0 +1,27 @@ +# Flags for GNU compiler +# CXX +set(OPS_CXXFLAGS "-fPIC -Wall -std=c++11 -g") +set(OPS_CXXFLAGS_RELEASE "-O3") +set(OPS_CXXFLAGS_DEBUG "-O0 -ffloat-store") +if(OpenMP_CXX_FOUND) + set(OPS_CXXFLAGS_RELEASE "${OPS_CXXFLAGS_RELEASE} -fopenmp") + # To test if GNU 14 is needed or 13 is OK with different set-up options + # Library compile the apps no + #if (CMAKE_CXX_COMPILER_VERSION GREATER_EQUAL "13") + # if(SET_CUDA_ARCH) + # #set(OPS_CXXFLAGS_OMPOFFLOAD "-fopenmp-simd" "-foffload-options=nvptx-none=\"march=sm_${SET_CUDA_ARCH}\"") + # # set(OPS_CXXFLAGS_OMPOFFLOAD "-fopenmp-simd" "-foffload=\"-march=sm_${SET_CUDA_ARCH}\"") + # #set(OPS_CXXFLAGS_OMPOFFLOAD "-fopenmp-simd") + # #set(OPS_CXXFLAGS_OMPOFFLOAD "-fopenmp-simd" "-march=sm_${SET_CUDA_ARCH}") + # endif() + #endif() +endif(OpenMP_CXX_FOUND) +# Fortran +if(OPS_FORTRAN) + set(OPS_FORTFLAGS "-g -ffree-form -ffree-line-length-none") + set(OPS_FORTFLAGS_RELEASE "-O3 -march=native") + set(OPS_FORTFLAGS_DEBUG "-O0 -ffloat-store -g3 -Og -ffpe-trap=invalid,zero -fcheck=all -fimplicit-none") + if(OpenMP_FORTRAN_FOUND) + set(OPS_FORTFLAGS_RELEASE "${OPS_FORTFLAGS_RELEASE} -fopenmp") + endif() +endif() diff --git a/cmake/compilers/OPS_flags_intel.cmake b/cmake/compilers/OPS_flags_intel.cmake new file mode 100644 index 0000000000..b733e66021 --- /dev/null +++ b/cmake/compilers/OPS_flags_intel.cmake @@ -0,0 +1,32 @@ +# Flags for Intel compiler +# CXX +set(OPS_CXXFLAGS "-std=c++17 -Wall -fPIC") +set(OPS_CXXFLAGS_RELEASE "-O3 -Ofast -march=native") +set(OPS_CXXFLAGS_DEBUG "-O0 -ffloat-store -g -DSYCL_COPY -DSYCL_USM") +if(IEEE) + set(OPS_CXXFLAGS_RELEASE "${OPS_CXXFLAGS_RELEASE} -fp-model=strict") +endif(IEEE) +if(OpenMP_CXX_FOUND) + set(OPS_CXXFLAGS_RELEASE "${OPS_CXXFLAGS_RELEASE} -qopenmp") +endif(OpenMP_CXX_FOUND) + +if (IntelDPCPP_FOUND) + # This is only for this specific target + # This flags are passed as target_compile_options + # Each flag must be append to the list + #set(OPS_CXXFLAGS_OMPOFFLOAD "-fiopenmp") + #list(APPEND OPS_CXXFLAGS_OMPOFFLOAD "-fopenmp-targets=spir64") + if(CUDAToolkit_FOUND) + # For NVIDIA + set(OPS_CXXFLAGS_SYCL "--fsycl") + list(APPEND OPS_CXXFLAGS_SYCL "-fsycl-targets=nvptx64-nvidia-cuda") + else() + # For INTEL GPUs + set(OPS_CXXFLAGS_SYCL "--fsycl") + list(APPEND OPS_CXXFLAGS_SYCL "-fsycl-targets=spir64") + endif() + # SYCL for CPU + #ifeq ($(GPU_VENDOR), UNKNOWN) + #SYCL_FLAGS := -fsycl -fsycl-targets=spir64_x86_64 + #OMPOFFLOADFLAGS := -fiopenmp -fopenmp-targets=spir64_x86_64 +endif() diff --git a/cmake/compilers/OPS_flags_msvc.cmake b/cmake/compilers/OPS_flags_msvc.cmake new file mode 100644 index 0000000000..4901e67979 --- /dev/null +++ b/cmake/compilers/OPS_flags_msvc.cmake @@ -0,0 +1,9 @@ +# Flags for MS compiler +# C +set(OPS_CFLAGS "-W1") +set(OPS_CFLAGS_RELEASE "/O2 /Ot") +set(OPS_CFLAGS_DEBUG "/Z7 /DEBUG") +# CXX +set(OPS_CXXFLAGS "-W1") +set(OPS_CXXFLAGS_RELEASE "/O2 /Ot") +set(OPS_CXXFLAGS_DEBUG "/Z7 /DEBUG") diff --git a/cmake/compilers/OPS_flags_nvhpc.cmake b/cmake/compilers/OPS_flags_nvhpc.cmake new file mode 100644 index 0000000000..2fc7a2f02e --- /dev/null +++ b/cmake/compilers/OPS_flags_nvhpc.cmake @@ -0,0 +1,21 @@ +# Flags for NVHPC compiler +# CXX +set(OPS_CXXFLAGS "-g") +set(OPS_CXXFLAGS_RELEASE "-O3 -fast -gopt -Xcudafe \"--diag_suppress=unrecognized_gcc_pragma\" ") +#set(OPS_CXXFLAGS_RELEASE "-O3 -fast -gopt ") +set(OPS_CXXFLAGS_DEBUG "-O0 ") +# OpenMP flag will be added by defualt if OpenMP is activated +# Here we look if an NVIDIA GPU has been found and we set the OMP GPU offload +if(SET_CUDA_ARCH) + # This is only for this specific target + # This flags are passed as target_compile_options + # Each flag must be append to the list + set(OPS_CXXFLAGS_OMPOFFLOAD "-mp") + list(APPEND OPS_CXXFLAGS_OMPOFFLOAD "-target=gpu") + list(APPEND OPS_CXXFLAGS_OMPOFFLOAD "-Minfo=accel") + #set(OPS_CXXFLAGS_OMPOFFLOAD "-mp=gpu -arch=sm_${SET_CUDA_ARCH}") + #set(OPS_CXXFLAGS_OMPOFFLOAD "-mp=gpu -gencode arch=compute_${SET_CUDA_ARCH},code=sm_${SET_CUDA_ARCH}") +endif() +if(IEEE) + set(OPS_CXXFLAGS_RELEASE "${OPS_CXXFLAGS_RELEASE} -Kieee -nofma") +endif(IEEE) diff --git a/cmake/FindNCCL.cmake b/cmake/packages/FindNCCL.cmake similarity index 100% rename from cmake/FindNCCL.cmake rename to cmake/packages/FindNCCL.cmake diff --git a/cmake/packages/OPS_CUDA.cmake b/cmake/packages/OPS_CUDA.cmake new file mode 100644 index 0000000000..919553d049 --- /dev/null +++ b/cmake/packages/OPS_CUDA.cmake @@ -0,0 +1,49 @@ +find_package(CUDAToolkit QUIET) + +if(CUDAToolkit_FOUND) + set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE}) + enable_language(CUDA) + enable_language(C) + if(NOT SET_GPU_ARCH) + include(FindCUDA/select_compute_arch) + CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1) + string(STRIP "${INSTALLED_GPU_CCS_1}" INSTALLED_GPU_CCS_2) + string(REPLACE " " ";" INSTALLED_GPU_CCS_3 "${INSTALLED_GPU_CCS_2}") + string(REPLACE "." "" CUDA_ARCH_LIST "${INSTALLED_GPU_CCS_3}") + set_property(GLOBAL PROPERTY CUDA_ARCHITECTURES "${CUDA_ARCH_LIST}") + message(STATUS "CUDA_ARCHITECTURES ${CUDA_ARCH_LIST} ${SET_GPU_ARCH}") + list(GET CUDA_ARCH_LIST 0 SET_CUDA_ARCH) + list(LENGTH CUDA_ARCH_LIST GPU_NUMBER) + message(STATUS "CUDA Architecture from autodetect ${SET_CUDA_ARCH}, N GPUS ${GPU_NUMBER}") + message(STATUS "If different architecture has to be used set -DSET_GPU_ARCH=XXX!") + endif() + set(CMAKE_CUDA_ARCHITECTURES "${SET_CUDA_ARCH}") + #set(CMAKE_CUDA_ARCHITECTURES "${SET_GPU_ARCH}" + # CACHE STRING "CUDA architectures") + # Set the CUDA Flags compilers + set(OPS_CUDAFLAGS "-Xcompiler=\"-fPIC\" -g -std=c++11 -gencode arch=compute_${SET_CUDA_ARCH},code=sm_${SET_CUDA_ARCH}") + + # If CMAKE_CUDA_ARCHITECTURES is correct there is no need to add the flags for the architectures, CMake does it + #set(OPS_CUDAFLAGS "-Xcompiler=\"-fPIC\" -g -std=c++11") + set(OPS_CUDAFLAGS_RELEASE "-O3 --use_fast_math") + set(OPS_CUDAFLAGS_DEBUG "-O0 -G") + if(IEEE) + set(OPS_CUDAFLAGS "${OPS_CUDAFLAGS} --fmad=false") + endif(IEEE) + + # Now look for nvidia-ml + find_library(NVML_LIB nvidia-ml + PATHS /usr/lib/x86_64-linux-gnu /usr/lib64) + + if(NOT NVML_LIB) + message(FATAL_ERROR "NVML not found. Install 'libnvidia-ml' or set NVML_ROOT.") + endif() + + add_library(NVML::nvml UNKNOWN IMPORTED) + set_target_properties(NVML::nvml PROPERTIES + IMPORTED_LOCATION "${NVML_LIB}") + +else() + message(WARNING "CUDA toolkit not found! The CUDA codes won't compile!") +endif() + diff --git a/cmake/packages/OPS_HIP.cmake b/cmake/packages/OPS_HIP.cmake new file mode 100644 index 0000000000..71bc1b5703 --- /dev/null +++ b/cmake/packages/OPS_HIP.cmake @@ -0,0 +1,70 @@ +# Search for rocm in common locations +#list(APPEND CMAKE_PREFIX_PATH /opt/rocm/hip /opt/rocm) +# Find hip +set(ENV{HIP_PLATFORM} "nvidia") +find_package(HIP QUIET) +if(HIP_FOUND) + message(STATUS "Found HIP: " ${HIP_VERSION}) +else() + message(FATAL_ERROR "Could not find HIP. Ensure that HIP is either installed in /opt/rocm/hip or the variable HIP_PATH is set to point to the right location.") +endif() +enable_language(HIP) +if(NOT DEFINED HIP_PATH) + if(NOT DEFINED ENV{HIP_PATH}) + set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed") + else() + set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed") + endif() +endif() + +if(NOT DEFINED ROCM_PATH) + if(DEFINED ENV{ROCM_PATH}) + set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCM has been installed") + elseif(DEFINED ENV{HIP_PATH}) + set(ROCM_PATH "$ENV{HIP_PATH}/.." CACHE PATH "Path to which ROCM has been installed") + else() + set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCM has been installed") + endif() +endif() + +if(NOT DEFINED HCC_PATH) + if(DEFINED ENV{HCC_PATH}) + set(HCC_PATH $ENV{HCC_PATH} CACHE PATH "Path to which HCC has been installed") + else() + set(HCC_PATH "${ROCM_PATH}/hcc" CACHE PATH "Path to which HCC has been installed") + endif() + set(HCC_HOME "${HCC_PATH}") +endif() + +if(NOT DEFINED HIP_CLANG_PATH) + if(NOT DEFINED ENV{HIP_CLANG_PATH}) + set(HIP_CLANG_PATH "${ROCM_PATH}/llvm/bin" CACHE PATH "Path to which HIP compatible clang binaries have been installed") + else() + set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH} CACHE PATH "Path to which HIP compatible clang binaries have been installed") + endif() +endif() + +set(CMAKE_MODULE_PATH "${HIP_PATH}/lib/cmake" "${HIP_PATH}/lib/cmake/hip" ${CMAKE_MODULE_PATH}) +list(APPEND CMAKE_PREFIX_PATH + "${HIP_PATH}/lib/cmake" +) +include(FindHIP) + +message(STATUS "CMAKE_MODULE_PATH in HIP set ${CMAKE_MODULE_PATH}") + +message(STATUS "HIP_PATH: ${HIP_PATH}") +message(STATUS "CMAKE_HIP_PLATFORM ${CMAKE_HIP_PLATFORM}") +message(STATUS "CMAKE_HIP_ARCHITECTURES ${CMAKE_HIP_ARCHITECTURES}") +message(STATUS "HIP_EXTENSIONS ${HIP_EXTENSIONS}") +#message(STATUS "Host and Device $ENV{hip::host} $ENV{hip::device}") +message(STATUS "ENV HIP PLATFORM: $ENV{HIP_PLATFORM}") +set(HIP_FOUND TRUE) + +if({$hip::host} STREQUAL "" OR {$hip::device} STREQUAL "") + message( + WARNING + "HIP support required but not found! You might need to use CMAKE_PREFIX_PATH to specify the path for HIP!" + ) + set(HIP_FOUND FALSE) +endif() +message(STATUS "Status of HIP_FOUND ${HIP_FOUND}") diff --git a/cmake/packages/OPS_MPI.cmake b/cmake/packages/OPS_MPI.cmake new file mode 100644 index 0000000000..1c6092cb13 --- /dev/null +++ b/cmake/packages/OPS_MPI.cmake @@ -0,0 +1,11 @@ +if(OPS_FORTRAN) + find_package(MPI QUIET COMPONENTS CXX Fortran) + message(STATUS "MPICXX FOUND ${MPI_CXX_FOUND}") + message(STATUS "MPIF90 FOUND ${MPI_Fortran_FOUND}") +else(OPS_FORTRAN) + find_package(MPI QUIET COMPONENTS CXX) + message(STATUS "MPICXX FOUND ${MPI_CXX_FOUND}") +endif() +if(NOT MPI_FOUND) + message(WARNING "MPI environment NOT FOUND! Only sequential code will be compiled!") +endif() diff --git a/cmake/packages/OPS_OMP.cmake b/cmake/packages/OPS_OMP.cmake new file mode 100644 index 0000000000..e445c13264 --- /dev/null +++ b/cmake/packages/OPS_OMP.cmake @@ -0,0 +1,23 @@ +#Check for OpenMP for CPU: this is activated by defauls with the exception of NVHPC above V23.1 + +message(STATUS "Set up OMP") +set(USE_OMP TRUE) +if (CMAKE_CXX_COMPILER_ID MATCHES "NVHPC") + message(STATUS "We are using NVHPC") + if (CMAKE_CXX_COMPILER_VERSION GREATER "23.1") + set(USE_OMP FALSE) + message(STATUS "OpenMP for multicore not enabled") + endif() +endif() + +option(OPS_OMP_CPU "Turn on OpenMP for multicore CPU" ON) +message(STATUS "Setting for option OMP_CPU ${OPS_OMP_CPU}") + +find_package(OpenMP QUIET) +if(NOT OpenMP_FOUND) + message(FATAL_ERROR "OpenMP support NOT FOUND!") +else() + message(STATUS "OpenMP version ${OpenMP_CXX_VERSION}") +endif() +find_package(OpenMP REQUIRED) + diff --git a/ops/CMakeLists.txt b/ops/CMakeLists.txt new file mode 100644 index 0000000000..7ba2787716 --- /dev/null +++ b/ops/CMakeLists.txt @@ -0,0 +1,31 @@ +# Add local module path +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +include(OPS_InstallMacros) + +# Includes to be done at level below to avoid issue of inclusion in the wrong target + +# Library build +add_subdirectory(c) +if (OPS_FORTRAN) + add_subdirectory(fortran) +endif() + +# Add the configure file +configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Config.cmake.in + "${CMAKE_BINARY_DIR}/OPSConfig.cmake" + INSTALL_DESTINATION ${ConfigPackageLocation}) + +# prepare the script for setting OPS environment variable variables +if(UNIX) + install( + CODE "file(WRITE \"${CMAKE_INSTALL_PREFIX}/bin/ops_env\" \"export OPS_INSTALL_PATH=${CMAKE_INSTALL_PREFIX}\n\")" + ) + + if(NOT ((CMAKE_INSTALL_PREFIX STREQUAL "/usr") + OR (CMAKE_INSTALL_PREFIX STREQUAL "/usr/local"))) + install( + CODE "file(APPEND \"${CMAKE_INSTALL_PREFIX}/bin/ops_env\" \"export PATH=$PATH:${CMAKE_INSTALL_PREFIX}/bin\n\")" + ) + endif() +endif() + diff --git a/ops/c/CMakeLists.txt b/ops/c/CMakeLists.txt index 10eb6e4e1b..5d4650858d 100644 --- a/ops/c/CMakeLists.txt +++ b/ops/c/CMakeLists.txt @@ -1,212 +1,35 @@ - - cmake_minimum_required(VERSION 3.18) - - project(API C CXX) - - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) - -# Prepare for files to compile the SEQ targets - file(GLOB_RECURSE CORE "${CMAKE_CURRENT_SOURCE_DIR}/src/core/*.cpp") - file(GLOB_RECURSE SEQ "${CMAKE_CURRENT_SOURCE_DIR}/src/sequential/*.cpp") - file(GLOB_RECURSE EXTERN "${CMAKE_CURRENT_SOURCE_DIR}/src/externlib/*.cpp") - - list(FILTER EXTERN EXCLUDE REGEX "hdf5") - list(FILTER CORE EXCLUDE REGEX "device") - - macro(InstallTarget Name Pos) - install( - TARGETS ops_${Name} - EXPORT ops_${Name}_targets - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - RUNTIME DESTINATION bin - INCLUDES - DESTINATION include) - install( - EXPORT ops_${Name}_targets - FILE ops_${Name}_targets.cmake - NAMESPACE OPS:: - DESTINATION ${Pos}) - endmacro() - -# TARGET: SEQ - add_library(ops_seq ${CORE} ${EXTERN} ${SEQ}) - - target_link_libraries(ops_seq PRIVATE OpenMP::OpenMP_CXX) - installtarget(seq ${ConfigPackageLocation}) - -# TARGET: HDF5_SEQ - if(HDF5_FOUND) - add_library( - ops_hdf5_seq ${CMAKE_CURRENT_SOURCE_DIR}/src/externlib/ops_hdf5_common.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/externlib/ops_hdf5.cpp) - target_link_libraries(ops_hdf5_seq PRIVATE hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - installtarget(hdf5_seq ${ConfigPackageLocation}) - endif() - -# TARGET: CUDA - if(CUDAToolkit_FOUND) - file(GLOB_RECURSE CUDA "${CMAKE_CURRENT_SOURCE_DIR}/src/cuda/*" - "${CMAKE_CURRENT_SOURCE_DIR}/src/core/ops_device_singlenode_common.cpp") - add_library(ops_cuda ${CORE} ${EXTERN} ${CUDA}) - target_link_libraries(ops_cuda PRIVATE OpenMP::OpenMP_CXX CUDA::cudart_static) - installtarget(cuda ${ConfigPackageLocation}) - - if(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(BUILD_OMPOFFLOAD TRUE) - set(OMPOFFLOAD_FLAGS -Minline=reshape -Mvect=levels:5 -mp=gpu -gpu=cc${CMAKE_CUDA_ARCHITECTURES}) - endif() - - endif() - -# TARGET: OMPOFFLOAD - if(BUILD_OMPOFFLOAD) - file(GLOB_RECURSE OMPOFFLOAD "${CMAKE_CURRENT_SOURCE_DIR}/src/ompoffload/*" - "${CMAKE_CURRENT_SOURCE_DIR}/src/core/ops_device_singlenode_common.cpp") - list(FILTER OMPOFFLOAD EXCLUDE REGEX "ops_ompoffload_common_omp5") - add_library(ops_ompoffload ${CORE} ${EXTERN} ${OMPOFFLOAD}) - target_link_libraries(ops_ompoffload PRIVATE OpenMP::OpenMP_CXX) - target_compile_options(ops_ompoffload PRIVATE ${OMPOFFLOAD_FLAGS}) - installtarget(ompoffload ${ConfigPackageLocation}) - endif() - -# TARGET: HIP - if(OPS_HIP) - if(HIP_FOUND) - file(GLOB_RECURSE HIP "${CMAKE_CURRENT_SOURCE_DIR}/src/hip/*.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/src/core/ops_device_singlenode_common.cpp") - add_library(ops_hip ${CORE} ${EXTERN} ${HIP}) - # -D__HIP_PLATFORM_NVCC__= -D__HIP_PLATFORM_NVIDIA__ - # -I/opt/rocm-4.5.0/hip/include -I/usr/local/cuda/include - target_compile_definitions(ops_hip PRIVATE OpenMP::OpenMP_CXX - __HIP_PLATFORM_NVIDIA__) - # target_compile_options(ops_hip PRIVATE -I/opt/rocm-4.5.0/hip/include) - target_link_libraries(ops_hip PRIVATE CUDA::cudart_static) - - target_include_directories(ops_hip PRIVATE /opt/rocm-4.5.0/hip/include) - installtarget(hip ${ConfigPackageLocation}) - endif() - endif() - - if(MPI_FOUND) - file(GLOB_RECURSE MPICORE "${CMAKE_CURRENT_SOURCE_DIR}/src/core/*.cpp") - list(FILTER MPICORE EXCLUDE REGEX "singlenode") - set(CommonFiles - "${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/ops_mpi_core.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/ops_mpi_decl.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/ops_mpi_partition.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/ops_mpi_rt_support.cpp") - -# TARGET: MPI - file(GLOB_RECURSE MPI - "${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/ops_mpi_rt_support_host.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/src/sequential/ops_host_common.cpp") - list(APPEND MPI ${CommonFiles}) - add_library(ops_mpi ${MPI} ${MPICORE} ${EXTERN}) - target_link_libraries(ops_mpi PRIVATE OpenMP::OpenMP_CXX MPI::MPI_CXX) - installtarget(mpi ${ConfigPackageLocation}) - -# TARGET: HDF5_MPI - if(HDF5_FOUND) - add_library(ops_hdf5_mpi ${CMAKE_CURRENT_SOURCE_DIR}/src/externlib/ops_hdf5_common.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/ops_mpi_hdf5.cpp) - target_link_libraries(ops_hdf5_mpi PRIVATE hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_CXX) - installtarget(hdf5_mpi ${ConfigPackageLocation}) - endif() - -# TARGET: MPI_CUDA - if(CUDAToolkit_FOUND) - file(GLOB_RECURSE MPICUDA "${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/*cuda*.cu" - "${CMAKE_CURRENT_SOURCE_DIR}/src/cuda/ops_cuda_common.cu") - list(FILTER MPICUDA EXCLUDE REGEX "singlenode") - list(APPEND MPICUDA ${CommonFiles}) - add_library(ops_mpi_cuda ${MPICORE} ${EXTERN} ${MPICUDA}) - target_link_libraries(ops_mpi_cuda PRIVATE OpenMP::OpenMP_CXX - CUDA::cudart_static MPI::MPI_CXX) - installtarget(mpi_cuda ${ConfigPackageLocation}) - - if(${CMAKE_C_COMPILER_ID} STREQUAL NVHPC) - set(BUILD_MPIOMPOFFLOAD TRUE) - endif() - endif() - -# TARGET: MPI_OMPOFFLOAD - if(BUILD_MPIOMPOFFLOAD) - file(GLOB_RECURSE MPIOMPOFFLOAD "${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/*ompoffload*.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/src/ompoffload/*") - list(FILTER MPIOMPOFFLOAD EXCLUDE REGEX "singlenode") - list(FILTER MPIOMPOFFLOAD EXCLUDE REGEX "ops_ompoffload_common_omp5") - list(APPEND MPIOMPOFFLOAD ${CommonFiles}) - add_library(ops_mpi_ompoffload ${MPICORE} ${EXTERN} ${MPIOMPOFFLOAD}) - target_link_libraries(ops_mpi_ompoffload PRIVATE OpenMP::OpenMP_CXX - MPI::MPI_CXX) - target_compile_options(ops_mpi_ompoffload PRIVATE ${OMPOFFLOAD_FLAGS}) - installtarget(mpi_ompoffload ${ConfigPackageLocation}) - endif() - -# TARGET: MPI_HIP - if(OPS_HIP) - if(HIP_FOUND) - file(GLOB_RECURSE MPIHIP "${CMAKE_CURRENT_SOURCE_DIR}/src/mpi/*hip*" - "${CMAKE_CURRENT_SOURCE_DIR}/src/hip/*") - list(FILTER MPIHIP EXCLUDE REGEX "singlenode") - list(APPEND MPIHIP ${CommonFiles}) - add_library(ops_mpi_hip ${MPICORE} ${EXTERN} ${MPIHIP}) - target_include_directories(ops_mpi_hip PRIVATE ${HIP_INCLUDE_DIRS}) - target_link_libraries(ops_mpi_hip PRIVATE OpenMP::OpenMP_CXX - ${HIP_LIBRARIES} MPI::MPI_CXX) - installtarget(mpi_hip ${ConfigPackageLocation}) - endif() - endif() - - endif() - - -# Tridiagonal library - if(LIBTRID_PATH) -# Sequential Tridiagonal Library - add_library(ops_trid_seq - ${CMAKE_CURRENT_SOURCE_DIR}/src/tridiag/ops_tridiag.cpp) - target_include_directories(ops_trid_seq PRIVATE ${LIBTRID_PATH}/include) - installtarget(trid_seq ${ConfigPackageLocation}) - -# Sequential CUDA Tridiagonal Library - if(CUDAToolkit_FOUND) - add_library(ops_trid_cuda - ${CMAKE_CURRENT_SOURCE_DIR}/src/tridiag/ops_tridiag_cuda.cpp) - target_include_directories(ops_trid_cuda PRIVATE ${LIBTRID_PATH}/include) - target_link_libraries(ops_trid_cuda PRIVATE CUDA::cudart_static) - installtarget(trid_cuda ${ConfigPackageLocation}) - endif() - - if(MPI_FOUND) -# MPI Tridiagonal Library - add_library(ops_trid_mpi - ${CMAKE_CURRENT_SOURCE_DIR}/src/tridiag/ops_tridiag_mpi.cpp) - target_include_directories(ops_trid_mpi PRIVATE ${LIBTRID_PATH}/include) - target_link_libraries(ops_trid_mpi PRIVATE MPI::MPI_CXX) - installtarget(trid_mpi ${ConfigPackageLocation}) - - if(CUDAToolkit_FOUND) -# MPI + CUDA Tridiagonal Library - add_library(ops_trid_mpi_cuda - ${CMAKE_CURRENT_SOURCE_DIR}/src/tridiag/ops_tridiag_mpi_cuda.cpp) - if(NCCL_FOUND) - target_compile_definitions(ops_trid_mpi_cuda PRIVATE -DTRID_NCCL) - target_include_directories(ops_trid_mpi_cuda - PRIVATE ${NCCL_INCLUDE_DIR}) - endif() - target_include_directories(ops_trid_mpi_cuda - PRIVATE ${LIBTRID_PATH}/include) - target_link_libraries(ops_trid_mpi_cuda PRIVATE CUDA::cudart_static - MPI::MPI_CXX) - installtarget(trid_mpi_cuda ${ConfigPackageLocation}) - endif() - endif() - endif() - - configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in - "${CMAKE_BINARY_DIR}/OPSConfig.cmake" - INSTALL_DESTINATION ${ConfigPackageLocation}) +# Add local module path +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +# Here we set all the lists for building the different targets +include(OPS_SET_SRC) +set(lib_prefix "ops_") + +# Add the includes +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) + +# Base build with sequenatial and MPI +include(OPS_BUILD_BASE) +# HDF5 IO +if(HDF5_FOUND) + include(OPS_BUILD_HDF5) +endif() +# seq with CUDA +if(CUDAToolkit_FOUND) + include(OPS_BUILD_CUDA) +endif() +## seq with HIP +#if(HIP_FOUND) +# message(STATUS "Call BUILD_HIP") +# include(OPS_BUILD_HIP) +#endif() +# OMPOFFLOAD +# If the FLAFS are set for the OMPOFFLOAD than we build +if (OPS_CXXFLAGS_OMPOFFLOAD) + include(OPS_BUILD_OMPOFFLOAD) +endif() + + +# Configure the installation of "include" +file(GLOB_RECURSE INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/include/*.h") +install(FILES ${INCLUDE} DESTINATION include) diff --git a/ops/c/cmake/OPS_BUILD_BASE.cmake b/ops/c/cmake/OPS_BUILD_BASE.cmake new file mode 100644 index 0000000000..d837c7db3a --- /dev/null +++ b/ops/c/cmake/OPS_BUILD_BASE.cmake @@ -0,0 +1,23 @@ +# Build the basic sequential target +set(TargetName "seq") +set(SRC ${CORE} ${EXTERN} ${SEQ}) +# +set(LibName "${lib_prefix}${TargetName}") +if (USE_OMP) + set(Links "OpenMP::OpenMP_CXX") +else () + set(Links "") +endif() +message(STATUS "Linking ${Links}") +set(Opts "") +set(Defs "") +set(Deps "") +setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +if(MPI_FOUND) + set(TargetName "mpi") + set(SRC ${MPICORE} ${MPICommonFiles} ${PUREMPI} ${EXTERN}) + # + set(LibName "${lib_prefix}${TargetName}") + list(APPEND Links "MPI::MPI_CXX") + setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +endif() diff --git a/ops/c/cmake/OPS_BUILD_CUDA.cmake b/ops/c/cmake/OPS_BUILD_CUDA.cmake new file mode 100644 index 0000000000..4df30272f8 --- /dev/null +++ b/ops/c/cmake/OPS_BUILD_CUDA.cmake @@ -0,0 +1,25 @@ +# Build the basic sequential target +set(TargetName "cuda") +set(SRC ${CORE} ${EXTERN} ${CUDA}) +# +set(LibName "${lib_prefix}${TargetName}") +if (USE_OMP) + set(Links "OpenMP::OpenMP_CXX" + "CUDA::cudart_static" + "NVML::nvml") +else () + set(Links "CUDA::cudart_static" + "NVML::nvml") +endif() +set(Opts "") +set(Defs "") +set(Deps "") +setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +if(MPI_FOUND) + set(TargetName "mpi_cuda") + set(SRC ${MPICORE} ${EXTERN} ${MPICUDA} ${MPICommonFiles}) + # + set(LibName "${lib_prefix}${TargetName}") + list(APPEND Links "MPI::MPI_CXX") + setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +endif() diff --git a/ops/c/cmake/OPS_BUILD_HDF5.cmake b/ops/c/cmake/OPS_BUILD_HDF5.cmake new file mode 100644 index 0000000000..08a3d9c6fd --- /dev/null +++ b/ops/c/cmake/OPS_BUILD_HDF5.cmake @@ -0,0 +1,20 @@ +# Build the basic sequential target +set(TargetName "hdf5_seq") +set(SRC ${HDF}) +# +set(LibName "${lib_prefix}${TargetName}") +set(Links "hdf5::hdf5" + "hdf5::hdf5_hl" + "MPI::MPI_CXX") +set(Opts "") +set(Defs "") +set(Deps "") +setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +if(MPI_FOUND) + set(TargetName "hdf5_mpi") + set(SRC ${HDF_MPI}) + # + set(LibName "${lib_prefix}${TargetName}") + setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +endif() + diff --git a/ops/c/cmake/OPS_BUILD_HIP.cmake b/ops/c/cmake/OPS_BUILD_HIP.cmake new file mode 100644 index 0000000000..21451a6ced --- /dev/null +++ b/ops/c/cmake/OPS_BUILD_HIP.cmake @@ -0,0 +1,66 @@ +message(STATUS "Build HIP") +# Build the basic sequential target +set(TargetName "hip") +set(SRC ${CORE} ${EXTERN} ${HIP}) +# +set(LibName "${lib_prefix}${TargetName}") +set(Links "OpenMP::OpenMP_CXX" + "hip::device") + +# For ROCm >=3.5, wipe hip-clang specific interface options which are propagated +set_target_properties(hip::device PROPERTIES INTERFACE_COMPILE_OPTIONS "-fPIC") +set_target_properties(hip::device PROPERTIES INTERFACE_LINK_LIBRARIES "hip::host") + +set(MY_HIPCC_OPTIONS "-fPIC") +set(MY_HCC_OPTIONS "") +set(MY_NVCC_OPTIONS "") +set(MY_CLANG_OPTIONS "") +set(STATIC_OR_SHARED STATIC) +set(Opts "") +set_source_files_properties(${SRC} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1) + +HIP_ADD_LIBRARY(${LibName} ${SRC}) + # HIPCC_OPTIONS "${MY_HIPCC_OPTIONS}" + # ${STATIC_OR_SHARED}) +#HIP_ADD_LIBRARY(${LibName} ${SRC} +# HIPCC_OPTIONS "${MY_HIPCC_OPTIONS}" +# HCC_OPTIONS "${MY_HCC_OPTIONS}" +# NVCC_OPTIONS "${MY_NVCC_OPTIONS}" +# CLANG_OPTIONS "${MY_CLANG_OPTIONS}" +# ${STATIC_OR_SHARED}) +set_target_properties(${LibName} PROPERTIES LINKER_LANGUAGE HIP) +foreach(Link IN LISTS Links) + target_link_libraries(${LibName} PRIVATE ${Link}) +endforeach() +target_link_libraries(${LibName} + PRIVATE rocrand + PRIVATE hiprand + PRIVATE rocsparse +) + +# Additional flags only for this target +foreach(Opt IN LISTS Opts) + target_compile_options(${LibName} PRIVATE ${Opt}) +endforeach() +installtarget(${LibName} ${ConfigPackageLocation}) + + +#if(MPI_FOUND) +# set(TargetName "mpi_hip") +# set(SRC ${MPICORE} ${EXTERN} ${MPIHIP} ${MPICommonFiles}) +# # +# set(LibName "${lib_prefix}${TargetName}") +# list(APPEND Links "MPI::MPI_CXX") +# HIP_ADD_LIBRARY(${LibName} ${SRC} +# HIPCC_OPTIONS "${MY_HIPCC_OPTIONS}" +# ${STATIC_OR_SHARED}) +# set_target_properties(${LibName} PROPERTIES LINKER_LANGUAGE HIP) +# foreach(Link IN LISTS Links) +# target_link_libraries(${LibName} PRIVATE ${Link}) +# endforeach() +# # Additional flags only for this target +# foreach(Opt IN LISTS Opts) +# target_compile_options(${LibName} PRIVATE ${Opt}) +# endforeach() +# installtarget(${LibName} ${ConfigPackageLocation}) +#endif() diff --git a/ops/c/cmake/OPS_BUILD_OMPOFFLOAD.cmake b/ops/c/cmake/OPS_BUILD_OMPOFFLOAD.cmake new file mode 100644 index 0000000000..e045abeae9 --- /dev/null +++ b/ops/c/cmake/OPS_BUILD_OMPOFFLOAD.cmake @@ -0,0 +1,23 @@ +# Build the basic sequential target +set(TargetName "ompoffload") +set(SRC ${CORE} ${EXTERN} ${OMPOFFLOAD}) +# +set(LibName "${lib_prefix}${TargetName}") +set(Links "OpenMP::OpenMP_CXX") +set(Opts "") +set(Defs "") +set(Deps "") +foreach(Flag IN LISTS OPS_CXXFLAGS_OMPOFFLOAD) + set(Opt "$<$:${Flag}>") + list(APPEND Opts "${Opt}") +endforeach() +setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +if(MPI_FOUND) + set(TargetName "mpi_ompoffload") + set(SRC ${MPICORE} ${MPICommonFiles} ${MPIOMPOFFLOAD} ${EXTERN}) + # + set(LibName "${lib_prefix}${TargetName}") + list(APPEND Links "MPI::MPI_CXX") + setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +endif() + diff --git a/ops/c/cmake/OPS_SET_SRC.cmake b/ops/c/cmake/OPS_SET_SRC.cmake new file mode 100644 index 0000000000..2f1cdb08bc --- /dev/null +++ b/ops/c/cmake/OPS_SET_SRC.cmake @@ -0,0 +1,51 @@ +# List the core subrotines +set(SRC_CXX_DIR "${CMAKE_SOURCE_DIR}/ops/c/src") +file(GLOB_RECURSE CORE "${SRC_CXX_DIR}/core/*.cpp") +file(GLOB_RECURSE SEQ "${SRC_CXX_DIR}/sequential/*.cpp") +file(GLOB_RECURSE EXTERN "${SRC_CXX_DIR}/externlib/*.cpp") +file(GLOB_RECURSE MPICORE "${SRC_CXX_DIR}/core/*.cpp") + +# Remove some specific routines for the lists +list(FILTER EXTERN EXCLUDE REGEX "hdf5") +list(FILTER CORE EXCLUDE REGEX "device") +list(FILTER MPICORE EXCLUDE REGEX "singlenode") +# Set the source for HDF5 +set(HDF "${SRC_CXX_DIR}/externlib/ops_hdf5_common.cpp" + "${SRC_CXX_DIR}/externlib/ops_hdf5.cpp") +set(HDF_MPI "${SRC_CXX_DIR}/externlib/ops_hdf5_common.cpp" + "${SRC_CXX_DIR}/mpi/ops_mpi_hdf5.cpp") +# Set the sources for CUDA +file(GLOB_RECURSE CUDA "${SRC_CXX_DIR}/cuda/*" + "${SRC_CXX_DIR}/core/ops_device_singlenode_common.cpp") +# Set sources for OMPOFFLOAD +file(GLOB_RECURSE OMPOFFLOAD "${SRC_CXX_DIR}/ompoffload/ops_ompoffload_rt_support_kernels.cpp" + "${SRC_CXX_DIR}/ompoffload/ops_ompoffload_singlenode.cpp" + "${SRC_CXX_DIR}/core/ops_device_singlenode_common.cpp") +if (OpenMP_CXX_VERSION GREATER_EQUAL "5") + list(APPEND OMPOFFLOAD "${SRC_CXX_DIR}/ompoffload/ops_ompoffload_common_omp5.cpp") +else() + list(APPEND OMPOFFLOAD "${SRC_CXX_DIR}/ompoffload/ops_ompoffload_common_omp4.cpp") +endif() +# +set(MPICommonFiles + "${SRC_CXX_DIR}/mpi/ops_mpi_core.cpp" + "${SRC_CXX_DIR}/mpi/ops_mpi_decl.cpp" + "${SRC_CXX_DIR}/mpi/ops_mpi_partition.cpp" + "${SRC_CXX_DIR}/mpi/ops_mpi_rt_support.cpp") + +file(GLOB_RECURSE PUREMPI + "${SRC_CXX_DIR}/mpi/ops_mpi_rt_support_host.cpp" + "${SRC_CXX_DIR}/sequential/ops_host_common.cpp") +# +file(GLOB_RECURSE MPICUDA "${SRC_CXX_DIR}/mpi/*cuda*.cu" + "${SRC_CXX_DIR}/cuda/*") +list(FILTER MPICUDA EXCLUDE REGEX "singlenode") +# +file(GLOB_RECURSE MPIOMPOFFLOAD "${SRC_CXX_DIR}/mpi/*ompoffload*.cpp" + "${SRC_CXX_DIR}/ompoffload/ops_ompoffload_rt_support_kernels.cpp") + +if (OpenMP_CXX_VERSION GREATER_EQUAL "5") + list(APPEND MPIOMPOFFLOAD "${SRC_CXX_DIR}/ompoffload/ops_ompoffload_common_omp5.cpp") +else() + list(APPEND MPIOMPOFFLOAD "${SRC_CXX_DIR}/ompoffload/ops_ompoffload_common_omp4.cpp") +endif() diff --git a/ops/c/src/hip/ops_hip_rt_support_kernels.cpp b/ops/c/src/hip/ops_hip_rt_support_kernels.cpp index 6ba70b8902..6011b2c206 100644 --- a/ops/c/src/hip/ops_hip_rt_support_kernels.cpp +++ b/ops/c/src/hip/ops_hip_rt_support_kernels.cpp @@ -47,6 +47,7 @@ #include #include +#include #include #include diff --git a/ops/c/Config.cmake.in b/ops/cmake/Config.cmake.in similarity index 100% rename from ops/c/Config.cmake.in rename to ops/cmake/Config.cmake.in diff --git a/ops/cmake/OPS_InstallMacros.cmake b/ops/cmake/OPS_InstallMacros.cmake new file mode 100644 index 0000000000..65f03d3986 --- /dev/null +++ b/ops/cmake/OPS_InstallMacros.cmake @@ -0,0 +1,38 @@ +macro(InstallTarget Name Pos) + install( + TARGETS ${Name} + EXPORT ${Name}_targets + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + RUNTIME DESTINATION bin + INCLUDES + DESTINATION include) + install( + EXPORT ${Name}_targets + FILE ${Name}_targets.cmake + NAMESPACE OPS:: + DESTINATION ${Pos}) +endmacro() + +macro(SetLib LibName SRC Links Opts Defs Deps) + message(STATUS "Library to be set ${LibName}") + add_library(${LibName} ${SRC}) + # Additional flags only for this target + #foreach(Dep IN LISTS Deps) + # message(STATUS "DEP ${Dep}") + # add_dependencies(${LibName} ${Dep}) + #endforeach() + foreach(Link IN LISTS Links) + target_link_libraries(${LibName} PRIVATE ${Link}) + endforeach() + # Additional flags only for this target + foreach(Opt IN LISTS Opts) + target_compile_options(${LibName} PRIVATE ${Opt}) + endforeach() + # Additional flags only for this target + foreach(Def IN LISTS Defs) + message(STATUS "DEF ${Def}") + target_compile_definitions(${LibName} PRIVATE ${Def}) + endforeach() + installtarget(${LibName} ${ConfigPackageLocation}) +endmacro() diff --git a/ops/fortran/CMakeLists.txt b/ops/fortran/CMakeLists.txt index a8dfa5e3b3..87b54dced5 100644 --- a/ops/fortran/CMakeLists.txt +++ b/ops/fortran/CMakeLists.txt @@ -1,163 +1,30 @@ +# Add local module path +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../c/cmake") - cmake_minimum_required(VERSION 3.18) +# Add the additional Fortran Sources +include(OPS_SET_SRC) +include(OPS_SET_SRC_FORT) +set(lib_prefix "ops_for_") - project(API Fortran CXX) +# Add the includes +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../c/include) - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../c/include) +# Base build with sequential and MPI +include(OPS_BUILD_BASE_FORT) - include(CheckLanguage) - check_language(Fortran) +## HDF5 IO +if(HDF5_FOUND) + include(OPS_BUILD_HDF5_FORT) +endif() - if(CMAKE_Fortran_COMPILER) - message(STATUS "Fortran compiler: ${CMAKE_Fortran_COMPILER}") +# Configure the installation of "include" +file(GLOB_RECURSE INCLUDE_F "${CMAKE_CURRENT_SOURCE_DIR}/include/*.inc") +install(FILES ${INCLUDE_F} DESTINATION include) - set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_INSTALL_PREFIX}/mod) +set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/include) +install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/ + DESTINATION include + FILES_MATCHING PATTERN "*.mod") - if(${CMAKE_Fortran_COMPILER_ID} STREQUAL Intel) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O3 -g -xHost -qopenmp") - set(CMAKE_Fortran_FLAGS_DEBUG "-g -traceback -qopenmp") - set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ip -xHOST -qopenmp") - elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL GNU) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O3 -g -ffree-form -ffree-line-length-none -fopenmp") - set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g3 -ffree-form -ffree-line-length-none -fopenmp") - set(CMAKE_Fortran_FLAGS_RELEASE "-Ofast -march=native -ffree-form -ffree-line-length-none -fopenmp") - elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O3 -fast -gopt -mp") - else() - message( - WARNING - "There is no default flag for the chosen Fortran compiler!" - ) - endif() - - if(CMAKE_BUILD_TYPE STREQUAL "Release") - message(STATUS "CMAKE_Fortran_FLAGS_RELEASE: ${CMAKE_Fortran_FLAGS_RELEASE}") - elseif(CMAKE_BUILD_TYPE STREQUAL "Debug") - message(STATUS "CMAKE_Fortran_FLAGS_DEBUG: ${CMAKE_Fortran_FLAGS_DEBUG}") - elseif(CMAKE_BUILD_TYPE STREQUAL "") - message(STATUS "CMAKE_Fortran_FLAGS: ${CMAKE_Fortran_FLAGS}") - else() - message(STATUS "Unknown build type: ${CMAKE_BUILD_TYPE}") - endif() - -# Prepare for files to compile the SEQ targets - list(APPEND F_HDF5 - "${CMAKE_CURRENT_SOURCE_DIR}/src/ops_for_hdf5_declarations.F90") - - list(APPEND F_COMMON - "${CMAKE_CURRENT_SOURCE_DIR}/src/ops_for_declarations.F90" - "${CMAKE_CURRENT_SOURCE_DIR}/src/ops_for_rt_support.F90") - - file(GLOB_RECURSE CORE "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/core/*.cpp") - file(GLOB_RECURSE SEQ "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/sequential/*.cpp") - file(GLOB_RECURSE EXTERN "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/externlib/*.cpp") - - list(FILTER EXTERN EXCLUDE REGEX "hdf5") - list(FILTER CORE EXCLUDE REGEX "device") - - macro(InstallTarget Name Pos) - install( - TARGETS ops_${Name} - EXPORT ops_${Name}_targets - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - RUNTIME DESTINATION bin - INCLUDES - DESTINATION include) - install( - EXPORT ops_${Name}_targets - FILE ops_${Name}_targets.cmake - NAMESPACE OPS:: - DESTINATION ${Pos}) - endmacro() - - -# TARGET: SEQ - add_library(ops_for_seq ${CORE} ${SEQ} ${EXTERN} ${F_COMMON}) - target_compile_definitions(ops_for_seq PRIVATE OPS_FTN) - target_link_libraries(ops_for_seq PRIVATE OpenMP::OpenMP_CXX) - installtarget(for_seq ${ConfigPackageLocation}) - -# TARGET: HDF5_SEQ - if(HDF5_FOUND) - file(GLOB_RECURSE HDF5_SEQ - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/externlib/ops_hdf5_common.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/externlib/ops_hdf5.cpp") - add_library(ops_for_hdf5_seq ${HDF5_SEQ} ${F_HDF5}) - add_dependencies(ops_for_hdf5_seq ops_for_seq) - target_link_libraries(ops_for_hdf5_seq PRIVATE hdf5::hdf5 hdf5::hdf5_hl) - target_compile_definitions(ops_for_hdf5_seq PRIVATE OPS_FTN) - installtarget(for_hdf5_seq ${ConfigPackageLocation}) - endif() - -# TARGET: CUDA - if(CUDAToolkit_FOUND) - if(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC) - - list(APPEND CUDA - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/cuda/ops_cuda_common.cu" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/cuda/ops_cuda_rt_support.cu" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/cuda/ops_cuda_singlenode.cu" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/core/ops_device_singlenode_common.cpp") - add_library(ops_for_cuda ${CORE} ${EXTERN} ${CUDA} ${F_COMMON}) - target_compile_definitions(ops_for_cuda PRIVATE OPS_FTN) - installtarget(for_cuda ${ConfigPackageLocation}) - endif() - endif() - - if(MPI_FOUND) - file(GLOB_RECURSE MPICORE "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/core/*.cpp") - list(FILTER MPICORE EXCLUDE REGEX "singlenode") - set(CommonFiles - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/mpi/ops_mpi_core.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/mpi/ops_mpi_decl.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/mpi/ops_mpi_partition.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/mpi/ops_mpi_rt_support.cpp") - - -# TARGET: MPI - file(GLOB_RECURSE MPI_HOST - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/mpi/ops_mpi_rt_support_host.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/sequential/ops_host_common.cpp") - - list(APPEND MPI_HOST ${CommonFiles}) - add_library(ops_for_mpi ${MPI_HOST} ${MPICORE} ${EXTERN} ${F_COMMON}) - target_compile_definitions(ops_for_mpi PRIVATE OPS_FTN) - target_link_libraries(ops_for_mpi PRIVATE OpenMP::OpenMP_CXX MPI::MPI_CXX MPI::MPI_Fortran) - installtarget(for_mpi ${ConfigPackageLocation}) - - -# TARGET: HDF5_MPI - if(HDF5_FOUND) - file(GLOB_RECURSE HDF5_MPI - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/mpi/ops_mpi_hdf5.cpp" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/externlib/ops_hdf5_common.cpp") - - add_library(ops_for_hdf5_mpi ${HDF5_MPI} ${F_HDF5}) - add_dependencies(ops_for_hdf5_mpi ops_for_mpi) - target_compile_definitions(ops_for_hdf5_mpi PRIVATE OPS_FTN) - target_link_libraries(ops_for_hdf5_mpi PRIVATE hdf5::hdf5 hdf5::hdf5_hl - MPI::MPI_Fortran) - installtarget(for_hdf5_mpi ${ConfigPackageLocation}) - endif() - -# TARGET: MPI_CUDA - if(CUDAToolkit_FOUND) - if(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC) - - list(APPEND MPI_DEVICE - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/cuda/ops_cuda_common.cu" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/src/mpi/ops_mpi_rt_support_cuda.cu") - list(APPEND MPI_DEVICE ${CommonFiles}) - - add_library(ops_for_mpi_cuda ${MPI_DEVICE} ${MPICORE} ${EXTERN} ${F_COMMON}) - target_compile_definitions(ops_for_mpi_cuda PRIVATE OPS_FTN) - target_link_libraries(ops_for_mpi_cuda PRIVATE OpenMP::OpenMP_CXX MPI::MPI_CXX MPI::MPI_Fortran) - installtarget(for_mpi_cuda ${ConfigPackageLocation}) - endif() - endif() - - endif() - - endif(CMAKE_Fortran_COMPILER) diff --git a/ops/fortran/cmake/OPS_BUILD_BASE_FORT.cmake b/ops/fortran/cmake/OPS_BUILD_BASE_FORT.cmake new file mode 100644 index 0000000000..70e890b67c --- /dev/null +++ b/ops/fortran/cmake/OPS_BUILD_BASE_FORT.cmake @@ -0,0 +1,25 @@ +# Build the basic sequential target +set(TargetName "seq") +set(SRC ${CORE} ${EXTERN} ${SEQ}) +# +set(LibName "${lib_prefix}${TargetName}") +set(Links "ops_for_common") +if (USE_OMP) + set(Links "OpenMP::OpenMP_CXX") +else () + set(Links "") +endif() +message(STATUS "Linking ${Links}") +set(Opts "") +set(Defs "OPS_FTN") +set(Deps "") +setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +if(MPI_FOUND) + set(TargetName "mpi") + set(SRC ${MPICORE} ${MPICommonFiles} ${PUREMPI} ${EXTERN}) + # + set(LibName "${lib_prefix}${TargetName}") + list(APPEND Links "MPI::MPI_CXX") + list(APPEND Links "MPI::MPI_Fortran") + setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +endif() diff --git a/ops/fortran/cmake/OPS_BUILD_HDF5_FORT.cmake b/ops/fortran/cmake/OPS_BUILD_HDF5_FORT.cmake new file mode 100644 index 0000000000..2a34ad1bdd --- /dev/null +++ b/ops/fortran/cmake/OPS_BUILD_HDF5_FORT.cmake @@ -0,0 +1,35 @@ +# Build the basic fortran HDF5 +add_library(ops_for_hdf5_common ${F_HDF}) +#target_link_libraries(ops_for_hdf5_common PUBLIC ops_for_common) + +#set(TargetName "hdf5_seq") +#set(SRC ${HDF}) +## +#set(LibName "${lib_prefix}${TargetName}") +#set(Links "ops_for_common" +# "ops_for_hdf5_common" +# "ops_for_seq" +# "hdf5::hdf5" +# "hdf5::hdf5_hl" +# "MPI::MPI_CXX" +# "MPI::MPI_Fortran") +#set(Opts "") +#set(Defs "OPS_FTN") +#set(Deps "") +#setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +#if(MPI_FOUND) +# set(TargetName "hdf5_mpi") +# set(SRC ${HDF_MPI}) +# set(Links "ops_for_common" +# "ops_for_hdf5_common" +# "ops_for_mpi" +# "hdf5::hdf5" +# "hdf5::hdf5_hl" +# "MPI::MPI_CXX" +# "MPI::MPI_Fortran") +# set(Deps "") +# # +# set(LibName "${lib_prefix}${TargetName}") +# setlib(${LibName} "${SRC}" "${Links}" "${Opts}" "${Defs}" "${Deps}") +#endif() + diff --git a/ops/fortran/cmake/OPS_SET_SRC_FORT.cmake b/ops/fortran/cmake/OPS_SET_SRC_FORT.cmake new file mode 100644 index 0000000000..f8cf2cc2bb --- /dev/null +++ b/ops/fortran/cmake/OPS_SET_SRC_FORT.cmake @@ -0,0 +1,16 @@ +set(SRC_FORT_DIR "${CMAKE_SOURCE_DIR}/ops/fortran/src") + +# Here we set the sources and we create support libraries +# Support libraries are necessary to avoid race condition during // builds +file(GLOB_RECURSE F_COMMON "${SRC_FORT_DIR}/ops_for_declarations.F90" + "${SRC_FORT_DIR}/ops_for_rt_support.F90") +add_library(ops_for_common ${F_COMMON}) + +set(F_SUPPORT "${SRC_FORT_DIR}/ops_for_math_support.F90") +add_library(ops_for_math_support ${F_SUPPORT}) + +#set(F_CUDA "${SRC_FORT_DIR}/ops_for_cuda_reduction.CUF") +#add_library(ops_for_cuda_reduction ${F_SUPPORT}) + +# HDF5 is build afterwards +set(F_HDF "${SRC_FORT_DIR}/ops_for_hdf5_declarations.F90") diff --git a/ops_translator/CMakeLists.txt b/ops_translator/CMakeLists.txt index e4a262a4da..43b8fc69f4 100644 --- a/ops_translator/CMakeLists.txt +++ b/ops_translator/CMakeLists.txt @@ -1,37 +1,32 @@ -# Specifiy the cmake version range, currenty work under 3.18 - cmake_minimum_required(VERSION 3.18) - - project(OPS_TRANSLATOR) - # The following operations are OS dependent, we do UNIX first - if (UNIX) -# Copy the entire directory content to CMAKE_INSTALL_PREFIX/translator - execute_process(COMMAND mkdir -p ${CMAKE_INSTALL_PREFIX}/translator) - execute_process(COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_INSTALL_PREFIX}/translator/.) +if (UNIX) + # Copy the entire directory content to CMAKE_INSTALL_PREFIX/translator + execute_process(COMMAND mkdir -p ${CMAKE_INSTALL_PREFIX}/translator) + execute_process(COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_INSTALL_PREFIX}/translator/.) -# Set the execute permission to a script which will install python virtual environment and required python -# packages to use Clang/Fparser based code-gen - execute_process(COMMAND chmod a+x ${CMAKE_INSTALL_PREFIX}/translator/ops_translator/setup_venv_cmake.sh - RESULT_VARIABLE chmod_result - OUTPUT_VARIABLE chmod_output) + # Set the execute permission to a script which will install python virtual environment and required python + # packages to use Clang/Fparser based code-gen + execute_process(COMMAND chmod a+x ${CMAKE_INSTALL_PREFIX}/translator/ops_translator/setup_venv_cmake.sh + RESULT_VARIABLE chmod_result + OUTPUT_VARIABLE chmod_output) - if(NOT chmod_result EQUAL 0) - message(FATAL_ERROR "Error: Failed to set execute permission on setup_venv_cmake.sh: ${chmod_output}") - endif() + if(NOT chmod_result EQUAL 0) + message(FATAL_ERROR "Error: Failed to set execute permission on setup_venv_cmake.sh: ${chmod_output}") + endif() -# Install the required Python dependencies at CMAKE_INSTALL_PREFIX/translator/ops_translator/ops_venv - file(GLOB OPS_VENV ${CMAKE_INSTALL_PREFIX}/translator/ops_translator/ops_venv) - if(NOT OPS_VENV) - message(STATUS "Python Virtual Environment not found in ${OPS_VENV}, proceeding with installation") + # Install the required Python dependencies at CMAKE_INSTALL_PREFIX/translator/ops_translator/ops_venv + file(GLOB OPS_VENV ${CMAKE_INSTALL_PREFIX}/translator/ops_translator/ops_venv) + if(NOT OPS_VENV) + message(STATUS "Python Virtual Environment not found in ${OPS_VENV}, proceeding with installation") - execute_process(COMMAND /bin/bash ${CMAKE_INSTALL_PREFIX}/translator/ops_translator/setup_venv_cmake.sh ${CMAKE_INSTALL_PREFIX}/translator/ops_translator - RESULT_VARIABLE req_install_result - OUTPUT_VARIABLE req_install_output) - if(NOT req_install_result EQUAL 0) - message(FATAL_ERROR "Error: Failed to execute setup_venv_cmake.sh: ${req_install_output}") - endif() - else() - message(STATUS "Python Virtual Environment found, skipping installation") + execute_process(COMMAND /bin/bash ${CMAKE_INSTALL_PREFIX}/translator/ops_translator/setup_venv_cmake.sh ${CMAKE_INSTALL_PREFIX}/translator/ops_translator + RESULT_VARIABLE req_install_result + OUTPUT_VARIABLE req_install_output) + if(NOT req_install_result EQUAL 0) + message(FATAL_ERROR "Error: Failed to execute setup_venv_cmake.sh: ${req_install_output}") endif() + else() + message(STATUS "Python Virtual Environment found, skipping installation") + endif() - endif() # UNIX if end +endif() # UNIX if end diff --git a/ops_translator/setup_venv_cmake.sh b/ops_translator/setup_venv_cmake.sh index 7753bd3f6a..728e184839 100644 --- a/ops_translator/setup_venv_cmake.sh +++ b/ops_translator/setup_venv_cmake.sh @@ -1,7 +1,7 @@ #!/bin/bash #Do not run below command if virtualenv is already installed -#python3 -m pip install --user virtualenv +python3 -m pip install --user virtualenv OPS_INSTALL_PATH=$1