Merge remote-tracking branch 'official/develop' into feature/no-gpu-direct-support

Author: svalat

.github/workflows/idefix-ci-doc.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
       - name: install doxygen

.github/workflows/idefix-ci-jobs.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -47,7 +47,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -64,7 +64,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -87,7 +87,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -109,7 +109,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -125,7 +125,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -141,7 +141,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -163,7 +163,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -187,7 +187,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -203,7 +203,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -223,7 +223,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -237,7 +237,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false
@@ -255,7 +255,7 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Check out repo
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: recursive
           persist-credentials: false

.github/workflows/idefix-ci.yml

@@ -17,14 +17,14 @@ jobs:
     if: ${{ github.repository == 'idefix-code/idefix' || github.repository == 'glesur/idefix' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
-      - uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
           python-version: 3.x
-      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0
-      - uses: pre-commit-ci/lite-action@50143aaf27e2c42e75a5e06185a471d9582e89df # v1.0.0
+      - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
+      - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 # v1.1.0
         if: always()
 
   icc-jobs:

CMakeLists.txt

@@ -19,6 +19,7 @@ option(Idefix_PYTHON "Enable python bindings (requires pybind11)" OFF)
 option(Idefix_MPI_GPU_DIRECT "Enable usage of GPU direct to avoid CPU-GPU copies for MPI messages." ON)
 option(Idefix_MPI_GPU_FORCE_COPY "When GPU_DIRECT is disabled, force using transfers for validation purpose." OFF)
 mark_as_advanced(Idefix_MPI_GPU_FORCE_TRANSFERS)
+option(Idefix_SUPPRESS_FMA "Disable FMA (fused multiply-add) contraction/codegen. Useful for code validation across architectures." OFF)
 set(Idefix_PROBLEM_DIR "${CMAKE_BINARY_DIR}" CACHE STRING "Problem directory to build for.")
 set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag")
 set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file")
@@ -49,6 +50,8 @@ include(ReplaceIdefixSource)
 include(AddIdefixSource)
 include(SetIdefixProperty)
 include(SetRequiredBuildSettingsForGCC8)
+include(CheckHdf5ParallelSupport)
+include(SuppressFMA)
 
 #Idefix requires Cuda Lambdas (experimental)
 if(Kokkos_ENABLE_CUDA)
@@ -64,6 +67,8 @@ include_directories(${Kokkos_INCLUDE_DIRS_RET})
 # Add Idefix CXX Flags
 add_compile_options(${Idefix_CXX_FLAGS})
 
+
+
 # Add filesystem libraries for GCC8
 set_required_build_settings_for_GCC8()
 
@@ -124,12 +129,25 @@ if(Idefix_HDF5)
     PUBLIC src/output/xdmf.cpp
     PUBLIC src/output/xdmf.hpp
   )
-  find_package(HDF5 REQUIRED)
-  target_link_libraries(idefix "${HDF5_LIBRARIES}")
+  # Prefer imported targets (matches older working behavior), then fall back.
+  find_package(HDF5 QUIET COMPONENTS C)
+  if(NOT HDF5_FOUND)
+    find_package(HDF5 REQUIRED MODULE COMPONENTS C)
+  endif()
+
+  set(_idefix_hdf5_link_items "${HDF5_LIBRARIES}")
+  if(TARGET hdf5::hdf5)
+    set(_idefix_hdf5_link_items hdf5::hdf5)
+  elseif(TARGET HDF5::HDF5)
+    set(_idefix_hdf5_link_items HDF5::HDF5)
+  endif()
+
+  target_link_libraries(idefix ${_idefix_hdf5_link_items})
   message(STATUS "Found HDF5 include directories: ${HDF5_INCLUDE_DIRS}")
   target_include_directories(idefix PUBLIC "${HDF5_INCLUDE_DIRS}")
   if(Idefix_MPI)
-    if(NOT HDF5_IS_PARALLEL)
+    CheckHdf5ParallelSupport("${_idefix_hdf5_link_items}")
+    if(NOT IDEFIX_HDF5_IS_PARALLEL)
       message(FATAL_ERROR "Parallel HDF5 required for Idefix_MPI but the found HDF5 library does not support it")
     endif()
   endif()
@@ -178,15 +196,15 @@ endif()
 if(Idefix_EVOLVE_VECTOR_POTENTIAL)
   add_compile_definitions("EVOLVE_VECTOR_POTENTIAL")
 endif()
-#update version.hpp if possible
+
+# determine idefix version from git
 git_describe(GIT_SHA1)
 set(Idefix_VERSION ${Idefix_VERSION_MAJOR}.${Idefix_VERSION_MINOR}.${Idefix_VERSION_PATCH}-${GIT_SHA1})
-file(WRITE ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_GIT_COMMIT \"${GIT_SHA1}\"\n")
-file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_MAJOR \"${Idefix_VERSION_MAJOR}\"\n")
-file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_MINOR \"${Idefix_VERSION_MINOR}\"\n")
-file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION_PATCH \"${Idefix_VERSION_PATCH}\"\n")
-file(APPEND ${CMAKE_SOURCE_DIR}/src/version.hpp "#define IDEFIX_VERSION \"${Idefix_VERSION}\"\n")
-
+configure_file(
+    ${CMAKE_SOURCE_DIR}/src/version.h.in
+    ${CMAKE_BINARY_DIR}/build/generated/version.h
+    @ONLY
+)
 
 if(NOT ${Idefix_DEFS} STREQUAL "definitions.hpp")
   add_compile_definitions("DEFINITIONS_FILE=\"${Idefix_DEFS}\"")
@@ -280,6 +298,21 @@ target_include_directories(idefix PUBLIC
 
 target_link_libraries(idefix Kokkos::kokkos)
 
+# Generate header with compiler information (name, flags, path)
+configure_file(
+    ${CMAKE_SOURCE_DIR}/src/compiler_info.h.in
+    ${CMAKE_BINARY_DIR}/build/generated/compiler_info.h
+    @ONLY
+)
+# Make sure the generated header is on the include path
+target_include_directories(idefix PRIVATE ${CMAKE_BINARY_DIR}/build/generated)
+
+# disable FMA if needed
+if(Idefix_SUPPRESS_FMA)
+  message(STATUS "FMA (fused multiply-add) contraction/codegen is disabled")
+  target_suppress_fma(idefix)
+endif()
+
 message(STATUS "Idefix final configuration")
 if(Idefix_EVOLVE_VECTOR_POTENTIAL)
   message(STATUS "    MHD:  ${Idefix_MHD} (Vector potential)")

cmake/CheckHdf5ParallelSupport.cmake

@@ -0,0 +1,50 @@
+include(CheckCSourceCompiles)
+
+# Check whether the HDF5 library found by find_package(HDF5) supports parallel
+# (MPI-IO) access, and store the result in IDEFIX_HDF5_IS_PARALLEL in the
+# caller's scope.
+#
+# Usage: CheckHdf5ParallelSupport(<hdf5_link_items>)
+# where <hdf5_link_items> is the list of HDF5 targets/libraries to link against.
+#
+# Some CMake/HDF5 combinations do not reliably set HDF5_IS_PARALLEL (e.g. when
+# HDF5 is found via its CMake config package rather than the FindHDF5 module).
+# Combine metadata checks with a compile+link probe for the MPI-IO symbols to
+# reliably detect parallel HDF5.
+function(CheckHdf5ParallelSupport hdf5_link_items)
+  set(_idefix_hdf5_is_parallel FALSE)
+  if(HDF5_IS_PARALLEL OR HDF5_C_IS_PARALLEL)
+    set(_idefix_hdf5_is_parallel TRUE)
+  endif()
+
+  if(NOT _idefix_hdf5_is_parallel AND DEFINED HDF5_C_COMPILER_EXECUTABLE)
+    execute_process(
+      COMMAND "${HDF5_C_COMPILER_EXECUTABLE}" -showconfig
+      OUTPUT_VARIABLE _idefix_hdf5_showconfig
+      ERROR_QUIET
+    )
+    if(_idefix_hdf5_showconfig MATCHES "Parallel HDF5:[ \t]*yes")
+      set(_idefix_hdf5_is_parallel TRUE)
+    endif()
+  endif()
+
+  set(CMAKE_REQUIRED_INCLUDES ${HDF5_INCLUDE_DIRS} ${MPI_C_INCLUDE_DIRS})
+  set(CMAKE_REQUIRED_LIBRARIES ${hdf5_link_items} MPI::MPI_C)
+  check_c_source_compiles(
+    "#include <mpi.h>
+     #include <hdf5.h>
+     int main(void) {
+       hid_t plist = H5Pcreate(H5P_FILE_ACCESS);
+       H5Pset_fapl_mpio(plist, MPI_COMM_WORLD, MPI_INFO_NULL);
+       H5Pclose(plist);
+       return 0;
+     }"
+    IDEFIX_HDF5_HAS_MPI_IO
+  )
+
+  if(IDEFIX_HDF5_HAS_MPI_IO)
+    set(_idefix_hdf5_is_parallel TRUE)
+  endif()
+
+  set(IDEFIX_HDF5_IS_PARALLEL ${_idefix_hdf5_is_parallel} PARENT_SCOPE)
+endfunction()

cmake/SuppressFMA.cmake

@@ -0,0 +1,102 @@
+#[[============================================================================
+  SuppressFMA.cmake
+
+  Provides an option and a function to optionally disable fused
+  multiply-add (FMA) code generation / contraction for a Kokkos-based CXX
+  target.
+
+  Kokkos wraps the real device compiler behind nvcc_wrapper (CUDA) or
+  hipcc (HIP), which makes CMAKE_CXX_COMPILER_ID report the *underlying
+  host* compiler (e.g. "GNU" or "Clang") instead of "NVIDIA" or "Clang
+  as HIP". Backend detection therefore relies on the Kokkos_ENABLE_*
+  variables exported by KokkosConfig.cmake / set by the Kokkos build,
+  and only falls back to CMAKE_CXX_COMPILER_ID for the plain host
+  compilers (no CUDA/HIP backend active):
+
+    - Kokkos_ENABLE_HIP  ON -> AMD HIP  (hipcc, clang-based)  : -ffp-contract=off
+    - Kokkos_ENABLE_CUDA ON -> NVIDIA nvcc (via nvcc_wrapper)  : --fmad=false
+    - otherwise, CMAKE_CXX_COMPILER_ID selects among:
+        GNU        : gcc/g++
+        Intel      : classic icc/icpc
+        IntelLLVM  : Intel oneAPI icx/icpx
+        Clang      : LLVM clang++
+        AppleClang : Xcode clang++
+        NVHPC      : NVIDIA HPC SDK (nvc++), e.g. for OpenMPTarget/OpenACC
+
+  Usage (after find_package(Kokkos) so Kokkos_ENABLE_* are defined):
+    include(SuppressFMA.cmake)
+    add_library(mylib source.cpp)
+    target_link_libraries(mylib PUBLIC Kokkos::kokkos)
+    target_suppress_fma(mylib)
+
+============================================================================]]
+
+include_guard(GLOBAL)
+
+# Determine the CXX FMA-suppression flags for the active Kokkos backend /
+# CXX compiler. Returns the list of flags (possibly empty) via out_var.
+function(_fma_suppression_flags_cxx out_var)
+    set(flags "")
+    set(id "${CMAKE_CXX_COMPILER_ID}")
+
+    # Kokkos backend takes priority: nvcc_wrapper/hipcc hide the real
+    # device compiler from CMAKE_CXX_COMPILER_ID.
+    if(Kokkos_ENABLE_HIP)
+        set(flags "-ffp-contract=off")
+
+    elseif(Kokkos_ENABLE_CUDA)
+        set(flags "--fmad=false")
+
+    elseif(id STREQUAL "GNU")
+        include(CheckCXXCompilerFlag)
+        check_cxx_compiler_flag("-mno-fma" IDEFIX_HAS_MNO_FMA)
+        set(flags "-ffp-contract=off")
+        if(IDEFIX_HAS_MNO_FMA)
+            list(APPEND flags "-mno-fma")
+        endif()
+    elseif(id MATCHES "^(Clang|AppleClang|CrayClang)$")
+        set(flags "-ffp-contract=off")
+
+    elseif(id STREQUAL "Intel")
+        # Intel classic compiler
+        set(flags "-fp-model=precise" "-no-fma")
+
+    elseif(id STREQUAL "IntelLLVM")
+        # Intel oneAPI compiler (clang-based)
+        set(flags "-ffp-contract=off" "-fp-model=strict")
+
+    elseif(id STREQUAL "NVHPC")
+        # NVIDIA HPC SDK (formerly PGI), e.g. OpenMPTarget/OpenACC backend
+        set(flags "-Mnofma") # untested
+    endif()
+
+    set(${out_var} "${flags}" PARENT_SCOPE)
+endfunction()
+
+# target_suppress_fma(<target>)
+#
+# Applies compiler-specific FMA-suppression flags to <target>'s CXX
+# sources, but only when the SUPPRESS_FMA option is ON. Safe to call
+# unconditionally.
+function(target_suppress_fma target)
+
+    if(NOT TARGET ${target})
+        message(FATAL_ERROR "target_suppress_fma: '${target}' is not a target")
+    endif()
+
+    _fma_suppression_flags_cxx(cxx_flags)
+
+    if(cxx_flags)
+        foreach(flag IN LISTS cxx_flags)
+            target_compile_options(${target} PRIVATE
+                $<$<COMPILE_LANGUAGE:CXX>:${flag}>
+            )
+        endforeach()
+    else()
+        message(VERBOSE
+            "target_suppress_fma: no FMA-suppression flag known for "
+            "CXX compiler '${CMAKE_CXX_COMPILER_ID}' "
+            "(Kokkos_ENABLE_CUDA=${Kokkos_ENABLE_CUDA}, "
+            "Kokkos_ENABLE_HIP=${Kokkos_ENABLE_HIP}) (target ${target})")
+    endif()
+endfunction()