From 63483e84c2b2b7aed3f280416a9b6e6cf6bd2950 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 12 Sep 2026 19:41:31 -0500 Subject: [PATCH 1/2] Probe the node with a solver binary, not just syscheck --- .github/scripts/preflight.sh | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/scripts/preflight.sh b/.github/scripts/preflight.sh index ec07f7d52..307f08cb2 100755 --- a/.github/scripts/preflight.sh +++ b/.github/scripts/preflight.sh @@ -134,6 +134,42 @@ run_probe() { fi } +# Probe the node with a solver binary, which unlike syscheck is compiled from the +# same vectorised Fortran the tests run and so actually contains the instructions a +# microarchitecture mismatch trips on. pre_process reaches +# s_assign_default_values_to_user_inputs before it needs any input file, which is +# exactly where the observed SIGILL landed, so it faults on a mismatched node +# without a case directory. +# +# ONLY SIGILL counts. Run without a case, pre_process fails for a dozen ordinary +# reasons -- no input file, no restart data, a missing module -- and none of them +# say anything about the node. Treating any non-zero status as a fault here would +# exclude every healthy node in the cluster. 132 is 128+4, a child killed by +# SIGILL; bash reports signals that way, and mpirun/srun forward it. +isa_probe() { + isa_bin=$(find build/install -name pre_process -type f -printf '%T@ %p\n' 2>/dev/null \ + | sort -rn | head -1 | cut -d' ' -f2-) + [ -n "$isa_bin" ] || return 0 + + isa_rc=0 + if [ "${#launcher[@]}" -eq 0 ]; then + isa_out=$("$isa_bin" 2>&1) || isa_rc=$? + else + isa_out=$("${launcher[@]}" "$isa_bin" 2>&1) || isa_rc=$? + fi + + case "$isa_rc:$isa_out" in + 132:*|*:*"Illegal instruction"*) + echo "::error::Preflight failed on $node: $isa_bin died with SIGILL." + echo "This is an INFRASTRUCTURE fault, not a code or test failure: the binary holds" + echo "an instruction this node does not implement, so it was built elsewhere." + echo "MFC_FAULT_NODE=$node" + exit $EXIT_NODE_FAULT + ;; + esac + echo "Preflight: $isa_bin started here (status $isa_rc, not SIGILL); node accepted." +} + # ${arr[@]+"${arr[@]}"} rather than "${arr[@]}": under set -u, bash 3.2 (which is # what macOS ships) treats an empty array expansion as an unbound variable. run_probe ${launcher[@]+"${launcher[@]}"} @@ -155,6 +191,15 @@ esac printf '%s\n' "$probe_out" if [ "$probe_rc" -eq 0 ]; then + # syscheck proves the GPU, MPI and launcher work here, but it is a few hundred + # lines and does not use the wide vector instructions the solver does. A binary + # built on one microarchitecture and run on an older one therefore sails through + # this probe and dies later in the real work: over 2026-09-11..12 node + # atl1-1-01-002-28-0 SIGILLed in pre_process on five case-optimization benchmarks + # across four attempts while syscheck passed every time, and the node was only + # excluded by hand (#1865). Probe a solver binary too, so the machinery that + # already exists can catch that class on its own. + isa_probe echo "Preflight: $node passed." exit $EXIT_HEALTHY fi From 0e367c70c2afd566043b5bdcf7676619152a243f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 12 Sep 2026 21:53:12 -0500 Subject: [PATCH 2/2] Share one portable, device-aware binary lookup between both preflight probes --- .github/scripts/preflight.sh | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/scripts/preflight.sh b/.github/scripts/preflight.sh index 307f08cb2..2b017075f 100755 --- a/.github/scripts/preflight.sh +++ b/.github/scripts/preflight.sh @@ -56,17 +56,23 @@ fi # only does so on Phoenix -- and a stale binary compiled for a different # microarchitecture dies with SIGILL, which would be reported as a bad node and # get a perfectly healthy one excluded. -newest_syscheck() { - # ls -t rather than find -printf: -printf is GNU-only, and on a BSD find it - # fails into 2>/dev/null, so discovery silently returns nothing and every - # probe is skipped as "no syscheck binary". - find "$@" -name syscheck -type f -exec ls -t {} + 2>/dev/null | head -1 +# ls -t rather than find -printf: -printf is GNU-only, and on a BSD find it +# fails into 2>/dev/null, so discovery silently returns nothing and every probe +# is skipped as "no binary". +newest_install_binary() { + name=$1 + shift + find build/install "$@" -name "$name" -type f -exec ls -t {} + 2>/dev/null | head -1 } -syscheck_bin=$(newest_syscheck build/install -path "*${device}*") -if [ -z "$syscheck_bin" ]; then - syscheck_bin=$(newest_syscheck build/install) -fi +# Prefer this job's device, fall back to any. Used for both probe binaries. +newest_for_device() { + found=$(newest_install_binary "$1" -path "*${device}*") + [ -n "$found" ] || found=$(newest_install_binary "$1") + printf '%s\n' "$found" +} + +syscheck_bin=$(newest_for_device syscheck) if [ -z "$syscheck_bin" ]; then # Nothing to probe with. A missing binary is a build problem, not a bad @@ -147,8 +153,7 @@ run_probe() { # exclude every healthy node in the cluster. 132 is 128+4, a child killed by # SIGILL; bash reports signals that way, and mpirun/srun forward it. isa_probe() { - isa_bin=$(find build/install -name pre_process -type f -printf '%T@ %p\n' 2>/dev/null \ - | sort -rn | head -1 | cut -d' ' -f2-) + isa_bin=$(newest_for_device pre_process) [ -n "$isa_bin" ] || return 0 isa_rc=0