Merge master to pick up the NVHPC IPO fix (#1878)

Author: sbryngelson

.github/scripts/monitor_slurm_job.sh

@@ -208,6 +208,7 @@ while true; do
   fi
 
   if is_terminal_state "$state"; then
+    final_state="$state"
     echo "[$(date +%H:%M:%S)] Job $job_id reached terminal state: $state"
     break
   fi
@@ -308,16 +309,26 @@ case "$exit_code" in
     ;;
 esac
 
-# Check if job succeeded
-if [ "$exit_code" != "0:0" ]; then
-  echo "ERROR: Job $job_id failed with exit code $exit_code"
+# Check if job succeeded.
+#
+# Both the recorded state and the exit code have to agree. SLURM intermittently
+# reports State=FAILED alongside ExitCode=0:0 -- on Phoenix the same job that
+# printed "reached terminal state: FAILED" here then reported "completed
+# successfully" and went green with 23 failing tests still in its log. Five of
+# nine "successful" gpu-acc runs on master were hiding failures that way, which
+# made a green self-hosted job worthless as evidence.
+#
+# run_monitored_slurm_job.sh already re-checks the state via sacct, but only
+# when this script exits non-zero, so nothing verified it on the success path.
+if [ "$exit_code" != "0:0" ] || { [ -n "${final_state:-}" ] && [ "$final_state" != "COMPLETED" ]; }; then
+  echo "ERROR: Job $job_id failed (state=${final_state:-unknown}, exit code $exit_code)"
   # A GPU memory fault explains itself in a block the test harness prints; lift
   # it onto the summary page so the faulting kernel and source line are visible
   # without opening the log at all.
   if grep -q 'GPU fault summary' "$output_file" 2>/dev/null; then
     ci_summary "### GPU memory fault\n\n\`\`\`\n$(grep -A6 'GPU fault summary' "$output_file" | head -8 | sed 's/`/'"'"'/g')\n\`\`\`\n"
   else
-    ci_summary "### Job \`$job_id\` failed (exit $exit_code)\n\n\`\`\`\n$(tail -n 15 "$output_file" | sed 's/`/'"'"'/g')\n\`\`\`\n"
+    ci_summary "### Job \`$job_id\` failed (state ${final_state:-unknown}, exit $exit_code)\n\n\`\`\`\n$(tail -n 15 "$output_file" | sed 's/`/'"'"'/g')\n\`\`\`\n"
   fi
   exit 1
 fi

.github/scripts/preflight.sh

@@ -56,17 +56,23 @@ fi
 # only does so on Phoenix -- and a stale binary compiled for a different
 # microarchitecture dies with SIGILL, which would be reported as a bad node and
 # get a perfectly healthy one excluded.
-newest_syscheck() {
-    # ls -t rather than find -printf: -printf is GNU-only, and on a BSD find it
-    # fails into 2>/dev/null, so discovery silently returns nothing and every
-    # probe is skipped as "no syscheck binary".
-    find "$@" -name syscheck -type f -exec ls -t {} + 2>/dev/null | head -1
+# ls -t rather than find -printf: -printf is GNU-only, and on a BSD find it
+# fails into 2>/dev/null, so discovery silently returns nothing and every probe
+# is skipped as "no binary".
+newest_install_binary() {
+    name=$1
+    shift
+    find build/install "$@" -name "$name" -type f -exec ls -t {} + 2>/dev/null | head -1
 }
 
-syscheck_bin=$(newest_syscheck build/install -path "*${device}*")
-if [ -z "$syscheck_bin" ]; then
-    syscheck_bin=$(newest_syscheck build/install)
-fi
+# Prefer this job's device, fall back to any. Used for both probe binaries.
+newest_for_device() {
+    found=$(newest_install_binary "$1" -path "*${device}*")
+    [ -n "$found" ] || found=$(newest_install_binary "$1")
+    printf '%s\n' "$found"
+}
+
+syscheck_bin=$(newest_for_device syscheck)
 
 if [ -z "$syscheck_bin" ]; then
     # Nothing to probe with. A missing binary is a build problem, not a bad
@@ -134,6 +140,41 @@ run_probe() {
     fi
 }
 
+# Probe the node with a solver binary, which unlike syscheck is compiled from the
+# same vectorised Fortran the tests run and so actually contains the instructions a
+# microarchitecture mismatch trips on. pre_process reaches
+# s_assign_default_values_to_user_inputs before it needs any input file, which is
+# exactly where the observed SIGILL landed, so it faults on a mismatched node
+# without a case directory.
+#
+# ONLY SIGILL counts. Run without a case, pre_process fails for a dozen ordinary
+# reasons -- no input file, no restart data, a missing module -- and none of them
+# say anything about the node. Treating any non-zero status as a fault here would
+# exclude every healthy node in the cluster. 132 is 128+4, a child killed by
+# SIGILL; bash reports signals that way, and mpirun/srun forward it.
+isa_probe() {
+    isa_bin=$(newest_for_device pre_process)
+    [ -n "$isa_bin" ] || return 0
+
+    isa_rc=0
+    if [ "${#launcher[@]}" -eq 0 ]; then
+        isa_out=$("$isa_bin" 2>&1) || isa_rc=$?
+    else
+        isa_out=$("${launcher[@]}" "$isa_bin" 2>&1) || isa_rc=$?
+    fi
+
+    case "$isa_rc:$isa_out" in
+        132:*|*:*"Illegal instruction"*)
+            echo "::error::Preflight failed on $node: $isa_bin died with SIGILL."
+            echo "This is an INFRASTRUCTURE fault, not a code or test failure: the binary holds"
+            echo "an instruction this node does not implement, so it was built elsewhere."
+            echo "MFC_FAULT_NODE=$node"
+            exit $EXIT_NODE_FAULT
+            ;;
+    esac
+    echo "Preflight: $isa_bin started here (status $isa_rc, not SIGILL); node accepted."
+}
+
 # ${arr[@]+"${arr[@]}"} rather than "${arr[@]}": under set -u, bash 3.2 (which is
 # what macOS ships) treats an empty array expansion as an unbound variable.
 run_probe ${launcher[@]+"${launcher[@]}"}
@@ -155,6 +196,15 @@ esac
 printf '%s\n' "$probe_out"
 
 if [ "$probe_rc" -eq 0 ]; then
+    # syscheck proves the GPU, MPI and launcher work here, but it is a few hundred
+    # lines and does not use the wide vector instructions the solver does. A binary
+    # built on one microarchitecture and run on an older one therefore sails through
+    # this probe and dies later in the real work: over 2026-09-11..12 node
+    # atl1-1-01-002-28-0 SIGILLed in pre_process on five case-optimization benchmarks
+    # across four attempts while syscheck passed every time, and the node was only
+    # excluded by hand (#1865). Probe a solver binary too, so the machinery that
+    # already exists can catch that class on its own.
+    isa_probe
     echo "Preflight: $node passed."
     exit $EXIT_HEALTHY
 fi

.github/scripts/submit-slurm-job.sh

@@ -159,7 +159,7 @@ elif [ "$device" = "gpu" ]; then
 #SBATCH -p $gpu_partition
 #SBATCH --ntasks-per-node=4
 #SBATCH -G${gpu_count}"
-            node_exclude="atl1-1-03-007-29-0,atl1-1-03-007-31-0"
+            node_exclude="atl1-1-03-007-29-0,atl1-1-03-007-31-0,atl1-1-01-002-28-0"
             ;;
         frontier|frontier_amd)
             sbatch_device_opts="\

cmake/MFCTargets.cmake

@@ -57,11 +57,20 @@ exit 0
         add_dependencies(${ARGS_TARGET} ${ARGS_TARGET}_lib)
         target_compile_options(${ARGS_TARGET} PRIVATE -Minline=lib:${ARGS_TARGET}_lib,except:f_is_default,except:s_compute_dt,except:my_inquire,except:s_mpi_abort,except:s_mpi_barrier,except:s_prohibit_abort,except:s_int_to_str,except:s_associate_cbc_coefficients_pointers)
 
-        # Exclude m_start_up and m_cbc from cross-file inlining: these are
-        # initialization/boundary code that trigger NVHPC 25.x fort2 ICE when
-        # too many functions are cross-inlined into them. GPU hot-path files
+        # Exclude these files from cross-file inlining. GPU hot-path files
         # (m_rhs, m_riemann_solvers, m_viscous, m_weno, etc.) keep full IPO.
-        foreach(_no_inline_file m_start_up m_cbc)
+        #
+        #   m_start_up, m_cbc      initialization/boundary code that trigger
+        #                          NVHPC 25.x fort2 ICEs when too many functions
+        #                          are cross-inlined into them.
+        #   m_pressure_relaxation  worse than an ICE, because it compiles: inlining
+        #                          the equation-of-state chain (s_phase_coefficients
+        #                          -> s_eos_coefficients -> s_reference_curve,
+        #                          s_phase_internal_energy, f_pressure) into the
+        #                          six-equation relaxation kernel silently drops the
+        #                          internal-energy update, leaving alpha and
+        #                          alpha_rho correct and int_en zero.
+        foreach(_no_inline_file m_start_up m_cbc m_pressure_relaxation)
             set_source_files_properties(
                 "${CMAKE_BINARY_DIR}/fypp/${ARGS_TARGET}/${_no_inline_file}.fpp.f90"
                 TARGET_DIRECTORY ${ARGS_TARGET}

docs/documentation/case.md

@@ -1343,6 +1343,9 @@ The entries labeled "Characteristic." are characteristic boundary conditions bas
 | `bc_[x,y,z]%%grcbc_out`        | Logical | Enable grcbc for subsonic outflow (pressure)|
 | `bc_[x,y,z]%%grcbc_vel_out`    | Logical | Enable grcbc for subsonic outflow (pressure + normal velocity) |
 | `bc_[x,y,z]%%vel_in`           | Real Array | Inflow velocities in x, y and z directions |
+| `bc_[x,y,z]%%vel_in_ramp`      | Real | Duration of a smooth start-up of the inflow velocity (0 = none) |
+| `bc_[x,y,z]%%vel_in_t0`        | Real | Time at which that ramp begins |
+| `bc_[x,y,z]%%vel_in_frac0`     | Real | Fraction of the final inflow velocity held before the ramp |
 | `bc_[x,y,z]%%vel_out`          | Real Array | Outflow velocities in x, y and z directions |
 | `bc_[x,y,z]%%pres_in`          | Real    | Inflow pressure |
 | `bc_[x,y,z]%%pres_out`         | Real    | Outflow pressure |
@@ -1351,6 +1354,8 @@ The entries labeled "Characteristic." are characteristic boundary conditions bas
 
 This boundary condition can be used for subsonic inflow (`bc_[x,y,z]%[beg,end]` = -7) and subsonic outflow (`bc_[x,y,z]%[beg,end]` = -8) characteristic boundary conditions. These are based on \cite Pirozzoli13. This enables to provide inflow and outflow conditions outside the computational domain.
 
+`bc_[x,y,z]%%vel_in_ramp` starts the inflow smoothly instead of holding it constant, which is what a jet or a tunnel accelerating from rest requires: the start-up is the event of interest, not a transient to be discarded. The inflow velocity is scaled by \f$f(t) = f_0 + (1 - f_0)\left[1 + \tanh\left(6 (t - t_0)/\tau - 3\right)\right]/2\f$, with \f$\tau\f$ = `vel_in_ramp`, \f$t_0\f$ = `vel_in_t0` and \f$f_0\f$ = `vel_in_frac0`, so it leaves \f$f_0\f$ of the final velocity at \f$t_0\f$ and is within half a percent of it at \f$t_0 + \tau\f$. A boundary with `vel_in_ramp = 0` is held constant, as before.
+
 ### Patch types {#patch-types}
 
 | #    | Name                    | Dim.  | Smooth | Description |

src/common/include/shared_parallel_macros.fpp

@@ -8,6 +8,13 @@
 #:set USING_CCE = (MFC_COMPILER == CCE_COMPILER_ID)
 #:set USING_AMD = (MFC_COMPILER == AMD_COMPILER_ID)
 
+#! Fallback extents the USING_AMD guards substitute for device-global array bounds when case
+#! optimization is off. They are not independent: sys_size counts the species, so AMD_SYS_SIZE_MAX
+#! must cover 3*num_fluids + num_vels + 1 + AMD_NUM_SPECIES_MAX. Keep them here rather than as
+#! literals at each declaration, so raising one cannot silently outgrow the other.
+#:set AMD_NUM_SPECIES_MAX = 60
+#:set AMD_SYS_SIZE_MAX = 70
+
 #:def ASSERT_LIST(data, datatype)
     #:assert data is not None
     #:assert isinstance(data, list)

src/common/m_boundary_primitives.fpp

@@ -21,8 +21,27 @@ module m_boundary_primitives
     logical :: dirichlet_from_buffers = .false.
     $:GPU_DECLARE(create='[dirichlet_from_buffers]')
 
+    public :: f_vel_ramp
+
 contains
 
+    !> Velocity scaling for a GRCBC inflow that is ramping up: unity unless `bc_[x,y,z]%vel_in_ramp` is set, so an unramped case is
+    !! untouched. Takes the time as an argument so the caller can evaluate it on the device from `mytime`, which the time stepper
+    !! already places there, rather than computing it on the host and copying the result every Runge-Kutta stage.
+    pure function f_vel_ramp(tau, t0, frac0, t) result(f)
+
+        $:GPU_ROUTINE(parallelism='[seq]')
+        real(wp), intent(in) :: tau, t0, frac0, t
+        real(wp)             :: f
+
+        if (tau > 0._wp) then
+            f = frac0 + (1._wp - frac0)*0.5_wp*(1._wp + tanh(6._wp*(t - t0)/tau - 3._wp))
+        else
+            f = 1._wp
+        end if
+
+    end function f_vel_ramp
+
     !> Fill ghost cells by copying the nearest boundary cell value along the specified direction.
     subroutine s_ghost_cell_extrapolation(q_prim_vf, bc_dir, bc_loc, k, l, q_T_sf)
 

src/common/m_checker_common.fpp

@@ -54,7 +54,16 @@ contains
         #:if not MFC_CASE_OPTIMIZATION
             @:PROHIBIT(num_fluids > 3, "num_fluids <= 3 for AMDFLang when Case optimization is off")
             @:PROHIBIT((bubbles_euler .or. bubbles_lagrange) .and. nb > 3, "nb <= 3 for AMDFLang when Case optimization is off")
-            @:PROHIBIT(chemistry .and. num_species > 10, "num_species > 10 for AMDFLang when Case optimization is off")
+            @:PROHIBIT(chemistry .and. num_species > ${AMD_NUM_SPECIES_MAX}$, &
+                       & "num_species <= ${AMD_NUM_SPECIES_MAX}$ for AMDFLang when Case optimization is off")
+            ! The sys_size bound is not independent of the one above it. Chemistry pins num_fluids to 1, so with
+            ! num_vels <= 3 the species terminate sys_size at 2*1 + 3 + 1 + 60 = 66 for five equations and
+            ! 3*1 + 3 + 1 + 60 = 67 for six; 70 covers both with room, and hypoelastic stresses would add up to six
+            ! more. It had no check of its own while the species cap was ten, because sys_size could not then reach
+            ! the dimension(20) the guard gives every sys_size array; HLLC's star states have no other bound, so
+            ! raising one cap without the other overruns them with nothing to say so.
+            @:PROHIBIT(sys_size > ${AMD_SYS_SIZE_MAX}$, &
+                       & "sys_size <= ${AMD_SYS_SIZE_MAX}$ for AMDFLang when Case optimization is off")
         #:endif
 
     end subroutine s_check_amd

src/common/m_chemistry.fpp

@@ -116,8 +116,8 @@ contains
         real(wp)                                               :: rho, omega_m
 
         #:if not MFC_CASE_OPTIMIZATION and USING_AMD
-            real(wp), dimension(10) :: Ys
-            real(wp), dimension(10) :: omega
+            real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: Ys
+            real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: omega
         #:else
             real(wp), dimension(num_species) :: Ys
             real(wp), dimension(num_species) :: omega
@@ -175,7 +175,7 @@ contains
         real(wp), parameter :: stiff_target = 0.5_wp
 
         #:if not MFC_CASE_OPTIMIZATION and USING_AMD
-            real(wp), dimension(10) :: Ys, cdot, ddot, y0, prod0, Lloss, alp
+            real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: Ys, cdot, ddot, y0, prod0, Lloss, alp
         #:else
             real(wp), dimension(num_species) :: Ys, cdot, ddot, y0, prod0, Lloss, alp
         #:endif
@@ -327,10 +327,10 @@ contains
         type(scalar_field), intent(in)                         :: q_T_sf
 
         #:if not MFC_CASE_OPTIMIZATION and USING_AMD
-            real(wp), dimension(10) :: Xs_L, Xs_R, Xs_cell, Ys_L, Ys_R, Ys_cell
-            real(wp), dimension(10) :: mass_diffusivities_mixavg1, mass_diffusivities_mixavg2
-            real(wp), dimension(10) :: mass_diffusivities_mixavg_Cell, dXk_dxi, h_l, h_r, h_k
-            real(wp), dimension(10) :: Mass_Diffu_Flux, dYk_dxi
+            real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: Xs_L, Xs_R, Xs_cell, Ys_L, Ys_R, Ys_cell
+            real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: mass_diffusivities_mixavg1, mass_diffusivities_mixavg2
+            real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: mass_diffusivities_mixavg_Cell, dXk_dxi, h_l, h_r, h_k
+            real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: Mass_Diffu_Flux, dYk_dxi
         #:else
             real(wp), dimension(num_species) :: Xs_L, Xs_R, Xs_cell, Ys_L, Ys_R, Ys_cell
             real(wp), dimension(num_species) :: mass_diffusivities_mixavg1, mass_diffusivities_mixavg2

src/common/m_constants.fpp

@@ -22,7 +22,7 @@ module m_constants
     integer, parameter  :: dflt_int = -100          !< Default integer value
     integer, parameter  :: fourier_rings = 5        !< Fourier filter ring limit
     integer, parameter  :: num_fluids_max = 10      !< Maximum number of fluids in the simulation
-    integer, parameter  :: num_probes_max = 10      !< Maximum number of flow probes in the simulation
+    integer, parameter  :: num_probes_max = 64      !< Maximum number of flow probes in the simulation
     integer, parameter  :: num_patches_max = 10     !< Maximum number of IC patches
     integer, parameter  :: num_ib_airfoils_max = 5  !< Maximum number of ib_airfoil instances
     integer, parameter  :: num_stl_models_max = 10