compiler-dependent fma suppression directly from cmake using -DIdefix_SUPPRESS_FMA

Author: glesur

CMakeLists.txt

@@ -16,6 +16,7 @@ option(Idefix_DEBUG "Enable Idefix debug features (makes the code very slow)" OF
 option(Idefix_RUNTIME_CHECKS "Enable runtime sanity checks" OFF)
 option(Idefix_WERROR "Treat compiler warnings as errors" OFF)
 option(Idefix_PYTHON "Enable python bindings (requires pybind11)" OFF)
+option(Idefix_SUPPRESS_FMA "Disable FMA (fused multiply-add) contraction/codegen. Useful for code validation accross architectures." OFF)
 set(Idefix_PROBLEM_DIR "${CMAKE_BINARY_DIR}" CACHE STRING "Problem directory to build for.")
 set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag")
 set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file")
@@ -43,6 +44,7 @@ include(AddIdefixSource)
 include(SetIdefixProperty)
 include(SetRequiredBuildSettingsForGCC8)
 include(CheckHdf5ParallelSupport)
+include(SuppressFMA)
 
 #Idefix requires Cuda Lambdas (experimental)
 if(Kokkos_ENABLE_CUDA)
@@ -58,6 +60,8 @@ include_directories(${Kokkos_INCLUDE_DIRS_RET})
 # Add Idefix CXX Flags
 add_compile_options(${Idefix_CXX_FLAGS})
 
+
+
 # Add filesystem libraries for GCC8
 set_required_build_settings_for_GCC8()
 
@@ -278,6 +282,12 @@ configure_file(
 # Make sure the generated header is on the include path
 target_include_directories(idefix PRIVATE ${CMAKE_BINARY_DIR}/build/generated)
 
+# disable FMA if needed
+if(Idefix_SUPPRESS_FMA)
+  message(STATUS "FMA (fused multiply-add) contraction/codegen is disabled")
+  target_suppress_fma(idefix)
+endif()
+
 message(STATUS "Idefix final configuration")
 if(Idefix_EVOLVE_VECTOR_POTENTIAL)
   message(STATUS "    MHD:  ${Idefix_MHD} (Vector potential)")

cmake/SuppressFMA.cmake

@@ -0,0 +1,104 @@
+#[[============================================================================
+  SuppressFMA.cmake
+
+  Provides an option and a function to optionally disable fused
+  multiply-add (FMA) code generation / contraction for a Kokkos-based CXX
+  target.
+
+  Kokkos wraps the real device compiler behind nvcc_wrapper (CUDA) or
+  hipcc (HIP), which makes CMAKE_CXX_COMPILER_ID report the *underlying
+  host* compiler (e.g. "GNU" or "Clang") instead of "NVIDIA" or "Clang
+  as HIP". Backend detection therefore relies on the Kokkos_ENABLE_*
+  variables exported by KokkosConfig.cmake / set by the Kokkos build,
+  and only falls back to CMAKE_CXX_COMPILER_ID for the plain host
+  compilers (no CUDA/HIP backend active):
+
+    - Kokkos_ENABLE_HIP  ON -> AMD HIP  (hipcc, clang-based)  : -ffp-contract=off
+    - Kokkos_ENABLE_CUDA ON -> NVIDIA nvcc (via nvcc_wrapper)  : --fmad=false
+    - otherwise, CMAKE_CXX_COMPILER_ID selects among:
+        GNU        : gcc/g++
+        Intel      : classic icc/icpc
+        IntelLLVM  : Intel oneAPI icx/icpx
+        Clang      : LLVM clang++
+        AppleClang : Xcode clang++
+        NVHPC      : NVIDIA HPC SDK (nvc++), e.g. for OpenMPTarget/OpenACC
+
+  Usage (after find_package(Kokkos) so Kokkos_ENABLE_* are defined):
+    include(SuppressFMA.cmake)
+    add_library(mylib source.cpp)
+    target_link_libraries(mylib PUBLIC Kokkos::kokkos)
+    target_suppress_fma(mylib)
+
+  FMA suppression is only actually applied if the cache option
+  SUPPRESS_FMA is ON (default OFF), so the function can be called
+  unconditionally and toggled at configure time with:
+    cmake -DSUPPRESS_FMA=ON ..
+============================================================================]]
+
+include_guard(GLOBAL)
+
+option(SUPPRESS_FMA "Disable FMA (fused multiply-add) contraction/codegen for CXX where possible" OFF)
+
+# Determine the CXX FMA-suppression flags for the active Kokkos backend /
+# CXX compiler. Returns the list of flags (possibly empty) via out_var.
+function(_fma_suppression_flags_cxx out_var)
+    set(flags "")
+    set(id "${CMAKE_CXX_COMPILER_ID}")
+
+    # Kokkos backend takes priority: nvcc_wrapper/hipcc hide the real
+    # device compiler from CMAKE_CXX_COMPILER_ID.
+    if(Kokkos_ENABLE_HIP)
+        set(flags "-ffp-contract=off")
+
+    elseif(Kokkos_ENABLE_CUDA)
+        set(flags "--fmad=false")
+
+    elseif(id STREQUAL "GNU")
+        set(flags "-ffp-contract=off" "-mno-fma")
+
+    elseif(id MATCHES "^(Clang|AppleClang)$")
+        set(flags "-ffp-contract=off")
+
+    elseif(id STREQUAL "Intel")
+        # Intel classic compiler
+        set(flags "-fp-model=precise" "-no-fma")
+
+    elseif(id STREQUAL "IntelLLVM")
+        # Intel oneAPI compiler (clang-based)
+        set(flags "-ffp-contract=off" "-fp-model=strict")
+
+    elseif(id STREQUAL "NVHPC")
+        # NVIDIA HPC SDK (formerly PGI), e.g. OpenMPTarget/OpenACC backend
+        set(flags "-Mnofma") # untested
+    endif()
+
+    set(${out_var} "${flags}" PARENT_SCOPE)
+endfunction()
+
+# target_suppress_fma(<target>)
+#
+# Applies compiler-specific FMA-suppression flags to <target>'s CXX
+# sources, but only when the SUPPRESS_FMA option is ON. Safe to call
+# unconditionally.
+function(target_suppress_fma target)
+
+    if(NOT TARGET ${target})
+        message(FATAL_ERROR "target_suppress_fma: '${target}' is not a target")
+    endif()
+
+    _fma_suppression_flags_cxx(cxx_flags)
+
+    if(cxx_flags)
+        foreach(flag IN LISTS cxx_flags)
+            target_compile_options(${target} PRIVATE
+                $<$<COMPILE_LANGUAGE:CXX>:${flag}>
+            )
+        endforeach()
+    else()
+        message(VERBOSE
+            "target_suppress_fma: no FMA-suppression flag known for "
+            "CXX compiler '${CMAKE_CXX_COMPILER_ID}' "
+            "(Kokkos_ENABLE_CUDA=${Kokkos_ENABLE_CUDA}, "
+            "Kokkos_ENABLE_HIP=${Kokkos_ENABLE_HIP}) (target ${target})")
+    endif()
+endfunction()

pytools/idfx_test.py

@@ -225,21 +225,19 @@ def _genCmakeCommand(self, definitionFile=""):
 
         if self.cuda:
             comm.append("-DKokkos_ENABLE_CUDA=ON")
-            # disable fmad operations on Cuda to make it compatible with CPU arithmetics
-            comm.append("-DIdefix_CXX_FLAGS=--fmad=false")
             # disable Async cuda malloc for tests performed on old UCX implementations
             comm.append("-DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF")
 
         if self.intel:
             # disable fmad operations on Cuda to make it compatible with CPU arithmetics
-            comm.append("-DIdefix_CXX_FLAGS=-fp-model=strict")
             comm.append("-DCMAKE_CXX_COMPILER=icpx")
             comm.append("-DCMAKE_C_COMPILER=icx")
 
         if self.hip:
             comm.append("-DKokkos_ENABLE_HIP=ON")
-            # disable fmad operations on HIP to make it compatible with CPU arithmetics
-            comm.append("-DIdefix_CXX_FLAGS=-ffp-contract=off")
+
+        # disable FMA for testing so that we have the same results on CPU and GPU (otherwise, the results are not bitwise identical)
+        comm.append("-DIdefix_SUPPRESS_FMA=ON")
 
         # if we use single precision
         if self.single: