Skip to content

Commit fbf76a3

Browse files
sbryngelsonsbryngelson
committed
Merge remote-tracking branch 'origin/master' into tmp-1846
# Conflicts: # src/simulation/m_data_output.fpp
2 parents 42fb5c7 + aa4e458 commit fbf76a3

30 files changed

Lines changed: 361 additions & 371 deletions

‎.github/scripts/monitor_slurm_job.sh‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ while true; do
208208
fi
209209

210210
if is_terminal_state "$state"; then
211+
final_state="$state"
211212
echo "[$(date +%H:%M:%S)] Job $job_id reached terminal state: $state"
212213
break
213214
fi
@@ -308,16 +309,26 @@ case "$exit_code" in
308309
;;
309310
esac
310311

311-
# Check if job succeeded
312-
if [ "$exit_code" != "0:0" ]; then
313-
echo "ERROR: Job $job_id failed with exit code $exit_code"
312+
# Check if job succeeded.
313+
#
314+
# Both the recorded state and the exit code have to agree. SLURM intermittently
315+
# reports State=FAILED alongside ExitCode=0:0 -- on Phoenix the same job that
316+
# printed "reached terminal state: FAILED" here then reported "completed
317+
# successfully" and went green with 23 failing tests still in its log. Five of
318+
# nine "successful" gpu-acc runs on master were hiding failures that way, which
319+
# made a green self-hosted job worthless as evidence.
320+
#
321+
# run_monitored_slurm_job.sh already re-checks the state via sacct, but only
322+
# when this script exits non-zero, so nothing verified it on the success path.
323+
if [ "$exit_code" != "0:0" ] || { [ -n "${final_state:-}" ] && [ "$final_state" != "COMPLETED" ]; }; then
324+
echo "ERROR: Job $job_id failed (state=${final_state:-unknown}, exit code $exit_code)"
314325
# A GPU memory fault explains itself in a block the test harness prints; lift
315326
# it onto the summary page so the faulting kernel and source line are visible
316327
# without opening the log at all.
317328
if grep -q 'GPU fault summary' "$output_file" 2>/dev/null; then
318329
ci_summary "### GPU memory fault\n\n\`\`\`\n$(grep -A6 'GPU fault summary' "$output_file" | head -8 | sed 's/`/'"'"'/g')\n\`\`\`\n"
319330
else
320-
ci_summary "### Job \`$job_id\` failed (exit $exit_code)\n\n\`\`\`\n$(tail -n 15 "$output_file" | sed 's/`/'"'"'/g')\n\`\`\`\n"
331+
ci_summary "### Job \`$job_id\` failed (state ${final_state:-unknown}, exit $exit_code)\n\n\`\`\`\n$(tail -n 15 "$output_file" | sed 's/`/'"'"'/g')\n\`\`\`\n"
321332
fi
322333
exit 1
323334
fi

‎.github/scripts/preflight.sh‎

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,17 +56,23 @@ fi
5656
# only does so on Phoenix -- and a stale binary compiled for a different
5757
# microarchitecture dies with SIGILL, which would be reported as a bad node and
5858
# get a perfectly healthy one excluded.
59-
newest_syscheck() {
60-
# ls -t rather than find -printf: -printf is GNU-only, and on a BSD find it
61-
# fails into 2>/dev/null, so discovery silently returns nothing and every
62-
# probe is skipped as "no syscheck binary".
63-
find "$@" -name syscheck -type f -exec ls -t {} + 2>/dev/null | head -1
59+
# ls -t rather than find -printf: -printf is GNU-only, and on a BSD find it
60+
# fails into 2>/dev/null, so discovery silently returns nothing and every probe
61+
# is skipped as "no binary".
62+
newest_install_binary() {
63+
name=$1
64+
shift
65+
find build/install "$@" -name "$name" -type f -exec ls -t {} + 2>/dev/null | head -1
6466
}
6567

66-
syscheck_bin=$(newest_syscheck build/install -path "*${device}*")
67-
if [ -z "$syscheck_bin" ]; then
68-
syscheck_bin=$(newest_syscheck build/install)
69-
fi
68+
# Prefer this job's device, fall back to any. Used for both probe binaries.
69+
newest_for_device() {
70+
found=$(newest_install_binary "$1" -path "*${device}*")
71+
[ -n "$found" ] || found=$(newest_install_binary "$1")
72+
printf '%s\n' "$found"
73+
}
74+
75+
syscheck_bin=$(newest_for_device syscheck)
7076

7177
if [ -z "$syscheck_bin" ]; then
7278
# Nothing to probe with. A missing binary is a build problem, not a bad
@@ -134,6 +140,41 @@ run_probe() {
134140
fi
135141
}
136142

143+
# Probe the node with a solver binary, which unlike syscheck is compiled from the
144+
# same vectorised Fortran the tests run and so actually contains the instructions a
145+
# microarchitecture mismatch trips on. pre_process reaches
146+
# s_assign_default_values_to_user_inputs before it needs any input file, which is
147+
# exactly where the observed SIGILL landed, so it faults on a mismatched node
148+
# without a case directory.
149+
#
150+
# ONLY SIGILL counts. Run without a case, pre_process fails for a dozen ordinary
151+
# reasons -- no input file, no restart data, a missing module -- and none of them
152+
# say anything about the node. Treating any non-zero status as a fault here would
153+
# exclude every healthy node in the cluster. 132 is 128+4, a child killed by
154+
# SIGILL; bash reports signals that way, and mpirun/srun forward it.
155+
isa_probe() {
156+
isa_bin=$(newest_for_device pre_process)
157+
[ -n "$isa_bin" ] || return 0
158+
159+
isa_rc=0
160+
if [ "${#launcher[@]}" -eq 0 ]; then
161+
isa_out=$("$isa_bin" 2>&1) || isa_rc=$?
162+
else
163+
isa_out=$("${launcher[@]}" "$isa_bin" 2>&1) || isa_rc=$?
164+
fi
165+
166+
case "$isa_rc:$isa_out" in
167+
132:*|*:*"Illegal instruction"*)
168+
echo "::error::Preflight failed on $node: $isa_bin died with SIGILL."
169+
echo "This is an INFRASTRUCTURE fault, not a code or test failure: the binary holds"
170+
echo "an instruction this node does not implement, so it was built elsewhere."
171+
echo "MFC_FAULT_NODE=$node"
172+
exit $EXIT_NODE_FAULT
173+
;;
174+
esac
175+
echo "Preflight: $isa_bin started here (status $isa_rc, not SIGILL); node accepted."
176+
}
177+
137178
# ${arr[@]+"${arr[@]}"} rather than "${arr[@]}": under set -u, bash 3.2 (which is
138179
# what macOS ships) treats an empty array expansion as an unbound variable.
139180
run_probe ${launcher[@]+"${launcher[@]}"}
@@ -155,6 +196,15 @@ esac
155196
printf '%s\n' "$probe_out"
156197

157198
if [ "$probe_rc" -eq 0 ]; then
199+
# syscheck proves the GPU, MPI and launcher work here, but it is a few hundred
200+
# lines and does not use the wide vector instructions the solver does. A binary
201+
# built on one microarchitecture and run on an older one therefore sails through
202+
# this probe and dies later in the real work: over 2026-09-11..12 node
203+
# atl1-1-01-002-28-0 SIGILLed in pre_process on five case-optimization benchmarks
204+
# across four attempts while syscheck passed every time, and the node was only
205+
# excluded by hand (#1865). Probe a solver binary too, so the machinery that
206+
# already exists can catch that class on its own.
207+
isa_probe
158208
echo "Preflight: $node passed."
159209
exit $EXIT_HEALTHY
160210
fi

‎.github/scripts/submit-slurm-job.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ elif [ "$device" = "gpu" ]; then
159159
#SBATCH -p $gpu_partition
160160
#SBATCH --ntasks-per-node=4
161161
#SBATCH -G${gpu_count}"
162-
node_exclude="atl1-1-03-007-29-0,atl1-1-03-007-31-0"
162+
node_exclude="atl1-1-03-007-29-0,atl1-1-03-007-31-0,atl1-1-01-002-28-0"
163163
;;
164164
frontier|frontier_amd)
165165
sbatch_device_opts="\

‎cmake/MFCTargets.cmake‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,20 @@ exit 0
5757
add_dependencies(${ARGS_TARGET} ${ARGS_TARGET}_lib)
5858
target_compile_options(${ARGS_TARGET} PRIVATE -Minline=lib:${ARGS_TARGET}_lib,except:f_is_default,except:s_compute_dt,except:my_inquire,except:s_mpi_abort,except:s_mpi_barrier,except:s_prohibit_abort,except:s_int_to_str,except:s_associate_cbc_coefficients_pointers)
5959

60-
# Exclude m_start_up and m_cbc from cross-file inlining: these are
61-
# initialization/boundary code that trigger NVHPC 25.x fort2 ICE when
62-
# too many functions are cross-inlined into them. GPU hot-path files
60+
# Exclude these files from cross-file inlining. GPU hot-path files
6361
# (m_rhs, m_riemann_solvers, m_viscous, m_weno, etc.) keep full IPO.
64-
foreach(_no_inline_file m_start_up m_cbc)
62+
#
63+
# m_start_up, m_cbc initialization/boundary code that trigger
64+
# NVHPC 25.x fort2 ICEs when too many functions
65+
# are cross-inlined into them.
66+
# m_pressure_relaxation worse than an ICE, because it compiles: inlining
67+
# the equation-of-state chain (s_phase_coefficients
68+
# -> s_eos_coefficients -> s_reference_curve,
69+
# s_phase_internal_energy, f_pressure) into the
70+
# six-equation relaxation kernel silently drops the
71+
# internal-energy update, leaving alpha and
72+
# alpha_rho correct and int_en zero.
73+
foreach(_no_inline_file m_start_up m_cbc m_pressure_relaxation)
6574
set_source_files_properties(
6675
"${CMAKE_BINARY_DIR}/fypp/${ARGS_TARGET}/${_no_inline_file}.fpp.f90"
6776
TARGET_DIRECTORY ${ARGS_TARGET}

‎docs/documentation/case.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1358,6 +1358,9 @@ The entries labeled "Characteristic." are characteristic boundary conditions bas
13581358
| `bc_[x,y,z]%%grcbc_out` | Logical | Enable grcbc for subsonic outflow (pressure)|
13591359
| `bc_[x,y,z]%%grcbc_vel_out` | Logical | Enable grcbc for subsonic outflow (pressure + normal velocity) |
13601360
| `bc_[x,y,z]%%vel_in` | Real Array | Inflow velocities in x, y and z directions |
1361+
| `bc_[x,y,z]%%vel_in_ramp` | Real | Duration of a smooth start-up of the inflow velocity (0 = none) |
1362+
| `bc_[x,y,z]%%vel_in_t0` | Real | Time at which that ramp begins |
1363+
| `bc_[x,y,z]%%vel_in_frac0` | Real | Fraction of the final inflow velocity held before the ramp |
13611364
| `bc_[x,y,z]%%vel_out` | Real Array | Outflow velocities in x, y and z directions |
13621365
| `bc_[x,y,z]%%pres_in` | Real | Inflow pressure |
13631366
| `bc_[x,y,z]%%pres_out` | Real | Outflow pressure |
@@ -1366,6 +1369,8 @@ The entries labeled "Characteristic." are characteristic boundary conditions bas
13661369

13671370
This boundary condition can be used for subsonic inflow (`bc_[x,y,z]%[beg,end]` = -7) and subsonic outflow (`bc_[x,y,z]%[beg,end]` = -8) characteristic boundary conditions. These are based on \cite Pirozzoli13. This enables to provide inflow and outflow conditions outside the computational domain.
13681371

1372+
`bc_[x,y,z]%%vel_in_ramp` starts the inflow smoothly instead of holding it constant, which is what a jet or a tunnel accelerating from rest requires: the start-up is the event of interest, not a transient to be discarded. The inflow velocity is scaled by \f$f(t) = f_0 + (1 - f_0)\left[1 + \tanh\left(6 (t - t_0)/\tau - 3\right)\right]/2\f$, with \f$\tau\f$ = `vel_in_ramp`, \f$t_0\f$ = `vel_in_t0` and \f$f_0\f$ = `vel_in_frac0`, so it leaves \f$f_0\f$ of the final velocity at \f$t_0\f$ and is within half a percent of it at \f$t_0 + \tau\f$. A boundary with `vel_in_ramp = 0` is held constant, as before.
1373+
13691374
### Patch types {#patch-types}
13701375

13711376
| # | Name | Dim. | Smooth | Description |

‎src/common/include/shared_parallel_macros.fpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@
88
#:set USING_CCE = (MFC_COMPILER == CCE_COMPILER_ID)
99
#:set USING_AMD = (MFC_COMPILER == AMD_COMPILER_ID)
1010

11+
#! Fallback extents the USING_AMD guards substitute for device-global array bounds when case
12+
#! optimization is off. They are not independent: sys_size counts the species, so AMD_SYS_SIZE_MAX
13+
#! must cover 3*num_fluids + num_vels + 1 + AMD_NUM_SPECIES_MAX. Keep them here rather than as
14+
#! literals at each declaration, so raising one cannot silently outgrow the other.
15+
#:set AMD_NUM_SPECIES_MAX = 60
16+
#:set AMD_SYS_SIZE_MAX = 70
17+
1118
#:def ASSERT_LIST(data, datatype)
1219
#:assert data is not None
1320
#:assert isinstance(data, list)

‎src/common/m_boundary_primitives.fpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,27 @@ module m_boundary_primitives
2121
logical :: dirichlet_from_buffers = .false.
2222
$:GPU_DECLARE(create='[dirichlet_from_buffers]')
2323

24+
public :: f_vel_ramp
25+
2426
contains
2527

28+
!> Velocity scaling for a GRCBC inflow that is ramping up: unity unless `bc_[x,y,z]%vel_in_ramp` is set, so an unramped case is
29+
!! untouched. Takes the time as an argument so the caller can evaluate it on the device from `mytime`, which the time stepper
30+
!! already places there, rather than computing it on the host and copying the result every Runge-Kutta stage.
31+
pure function f_vel_ramp(tau, t0, frac0, t) result(f)
32+
33+
$:GPU_ROUTINE(parallelism='[seq]')
34+
real(wp), intent(in) :: tau, t0, frac0, t
35+
real(wp) :: f
36+
37+
if (tau > 0._wp) then
38+
f = frac0 + (1._wp - frac0)*0.5_wp*(1._wp + tanh(6._wp*(t - t0)/tau - 3._wp))
39+
else
40+
f = 1._wp
41+
end if
42+
43+
end function f_vel_ramp
44+
2645
!> Fill ghost cells by copying the nearest boundary cell value along the specified direction.
2746
subroutine s_ghost_cell_extrapolation(q_prim_vf, bc_dir, bc_loc, k, l, q_T_sf)
2847

‎src/common/m_checker_common.fpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,16 @@ contains
5454
#:if not MFC_CASE_OPTIMIZATION
5555
@:PROHIBIT(num_fluids > 3, "num_fluids <= 3 for AMDFLang when Case optimization is off")
5656
@:PROHIBIT((bubbles_euler .or. bubbles_lagrange) .and. nb > 3, "nb <= 3 for AMDFLang when Case optimization is off")
57-
@:PROHIBIT(chemistry .and. num_species > 10, "num_species > 10 for AMDFLang when Case optimization is off")
57+
@:PROHIBIT(chemistry .and. num_species > ${AMD_NUM_SPECIES_MAX}$, &
58+
& "num_species <= ${AMD_NUM_SPECIES_MAX}$ for AMDFLang when Case optimization is off")
59+
! The sys_size bound is not independent of the one above it. Chemistry pins num_fluids to 1, so with
60+
! num_vels <= 3 the species terminate sys_size at 2*1 + 3 + 1 + 60 = 66 for five equations and
61+
! 3*1 + 3 + 1 + 60 = 67 for six; 70 covers both with room, and hypoelastic stresses would add up to six
62+
! more. It had no check of its own while the species cap was ten, because sys_size could not then reach
63+
! the dimension(20) the guard gives every sys_size array; HLLC's star states have no other bound, so
64+
! raising one cap without the other overruns them with nothing to say so.
65+
@:PROHIBIT(sys_size > ${AMD_SYS_SIZE_MAX}$, &
66+
& "sys_size <= ${AMD_SYS_SIZE_MAX}$ for AMDFLang when Case optimization is off")
5867
#:endif
5968
6069
end subroutine s_check_amd

‎src/common/m_chemistry.fpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,8 @@ contains
116116
real(wp) :: rho, omega_m
117117

118118
#:if not MFC_CASE_OPTIMIZATION and USING_AMD
119-
real(wp), dimension(10) :: Ys
120-
real(wp), dimension(10) :: omega
119+
real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: Ys
120+
real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: omega
121121
#:else
122122
real(wp), dimension(num_species) :: Ys
123123
real(wp), dimension(num_species) :: omega
@@ -175,7 +175,7 @@ contains
175175
real(wp), parameter :: stiff_target = 0.5_wp
176176
177177
#:if not MFC_CASE_OPTIMIZATION and USING_AMD
178-
real(wp), dimension(10) :: Ys, cdot, ddot, y0, prod0, Lloss, alp
178+
real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: Ys, cdot, ddot, y0, prod0, Lloss, alp
179179
#:else
180180
real(wp), dimension(num_species) :: Ys, cdot, ddot, y0, prod0, Lloss, alp
181181
#:endif
@@ -327,10 +327,10 @@ contains
327327
type(scalar_field), intent(in) :: q_T_sf
328328
329329
#:if not MFC_CASE_OPTIMIZATION and USING_AMD
330-
real(wp), dimension(10) :: Xs_L, Xs_R, Xs_cell, Ys_L, Ys_R, Ys_cell
331-
real(wp), dimension(10) :: mass_diffusivities_mixavg1, mass_diffusivities_mixavg2
332-
real(wp), dimension(10) :: mass_diffusivities_mixavg_Cell, dXk_dxi, h_l, h_r, h_k
333-
real(wp), dimension(10) :: Mass_Diffu_Flux, dYk_dxi
330+
real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: Xs_L, Xs_R, Xs_cell, Ys_L, Ys_R, Ys_cell
331+
real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: mass_diffusivities_mixavg1, mass_diffusivities_mixavg2
332+
real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: mass_diffusivities_mixavg_Cell, dXk_dxi, h_l, h_r, h_k
333+
real(wp), dimension(${AMD_NUM_SPECIES_MAX}$) :: Mass_Diffu_Flux, dYk_dxi
334334
#:else
335335
real(wp), dimension(num_species) :: Xs_L, Xs_R, Xs_cell, Ys_L, Ys_R, Ys_cell
336336
real(wp), dimension(num_species) :: mass_diffusivities_mixavg1, mass_diffusivities_mixavg2

‎src/common/m_constants.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ module m_constants
2222
integer, parameter :: dflt_int = -100 !< Default integer value
2323
integer, parameter :: fourier_rings = 5 !< Fourier filter ring limit
2424
integer, parameter :: num_fluids_max = 10 !< Maximum number of fluids in the simulation
25-
integer, parameter :: num_probes_max = 10 !< Maximum number of flow probes in the simulation
25+
integer, parameter :: num_probes_max = 64 !< Maximum number of flow probes in the simulation
2626
integer, parameter :: num_patches_max = 10 !< Maximum number of IC patches
2727
integer, parameter :: num_ib_airfoils_max = 5 !< Maximum number of ib_airfoil instances
2828
integer, parameter :: num_stl_models_max = 10

0 commit comments

Comments
 (0)