From d0b6d5239e12f18615d0a96d419c8e9a1c55cd15 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 12 Jun 2026 08:11:08 -0400 Subject: [PATCH 1/6] ci: shard the case-optimization pre-build into parallel SLURM jobs --- .github/scripts/prebuild-case-optimization.sh | 51 ++++++++++++++++++- .github/workflows/test.yml | 21 +++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/.github/scripts/prebuild-case-optimization.sh b/.github/scripts/prebuild-case-optimization.sh index f580770330..a5869b7dfa 100755 --- a/.github/scripts/prebuild-case-optimization.sh +++ b/.github/scripts/prebuild-case-optimization.sh @@ -22,12 +22,30 @@ case "$cluster" in *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;; esac +# Optional sharding (format "i/N", e.g. "1/2"), set by submit-slurm-job.sh's +# [shard] argument via $job_shard: shard i builds every Nth case of the sorted +# case list. Unset = build all cases in one job (default; other clusters). +shard="${job_shard:-}" +if [ -n "$shard" ]; then + shard_idx="${shard%%/*}" + shard_count="${shard##*/}" + case "${shard_idx}${shard_count}" in + ''|*[!0-9]*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;; + esac + if [ "$shard" != "$shard_idx/$shard_count" ] || [ "$shard_idx" -lt 1 ] || [ "$shard_idx" -gt "$shard_count" ]; then + echo "ERROR: bad shard '$shard' (expected i/N with 1 <= i <= N)"; exit 1 + fi +fi + # Phoenix starts fresh (no prior dep build); other clusters pre-build deps via # build.sh first, so we must preserve them and only clean MFC target staging. +# Sharded jobs share one workspace and run concurrently, so the workflow +# cleans once before submitting them — cleaning here would wipe a sibling +# shard's in-progress build. if [ "$cluster" = "phoenix" ]; then source .github/scripts/clean-build.sh clean_build -else +elif [ -z "$shard" ]; then find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true fi @@ -40,7 +58,38 @@ case "$job_interface" in *) echo "ERROR: prebuild requires gpu interface (acc or omp)"; exit 1 ;; esac +# Case-optimized simulation builds land in per-case hash-named staging dirs, +# but syscheck/pre_process/post_process hash identically across these cases. +# Concurrent shards must not build those shared staging dirs simultaneously: +# shard 1 builds them first; the other shards wait for its marker, after +# which their builds no-op in the shared dirs. +if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then + shared_marker="build/.prebuild-shared-targets-done" + set -- benchmarks/*/case.py + first_case="$1" + if [ "$shard_idx" -eq 1 ]; then + echo "=== Shard 1/$shard_count: building shared targets ===" + ./mfc.sh build -i "$first_case" -t syscheck pre_process post_process --case-optimization $gpu_opts -j 8 + touch "$shared_marker" + else + echo "=== Shard $shard_idx/$shard_count: waiting for shard 1 to build shared targets ===" + waited=0 + until [ -f "$shared_marker" ]; do + if [ "$waited" -ge 5400 ]; then + echo "ERROR: timed out waiting for $shared_marker"; exit 1 + fi + sleep 30 + waited=$((waited + 30)) + done + fi +fi + +idx=0 for case in benchmarks/*/case.py; do + idx=$((idx + 1)) + if [ -n "$shard" ] && [ $(((idx - 1) % shard_count)) -ne $((shard_idx - 1)) ]; then + continue + fi echo "=== Pre-building: $case ===" ./mfc.sh run "$case" --case-optimization $gpu_opts -j 8 --dry-run done diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c8e4cc61fa..c53849e599 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -523,7 +523,22 @@ jobs: - name: Pre-Build (SLURM) if: matrix.cluster == 'frontier_amd' - run: bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} + # AMD flang is slow enough that one serial pre-build job exceeds its + # walltime, so split the case list across two concurrent SLURM jobs. + # The shards share this workspace and skip their in-job staging clean, + # so clean once here on the login node before submitting. + run: | + find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true + find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true + rm -f build/.prebuild-shared-targets-done + bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} 1/2 & + pid1=$! + bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }} 2/2 & + pid2=$! + rc=0 + wait "$pid1" || rc=1 + wait "$pid2" || rc=1 + exit $rc - name: Build & Run Case-Optimization Tests if: matrix.cluster != 'phoenix' && matrix.cluster != 'frontier_amd' @@ -546,6 +561,8 @@ jobs: if: always() run: | for f in prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out \ + prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}-1-of-2.out \ + prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}-2-of-2.out \ run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out; do [ -f "$f" ] && echo "=== $f ===" && cat "$f" done @@ -556,5 +573,5 @@ jobs: with: name: case-opt-${{ strategy.job-index }}-${{ matrix.cluster }}-${{ matrix.interface }} path: | - prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out + prebuild-case-optimization-${{ matrix.device }}-${{ matrix.interface }}*.out run-case-optimization-${{ matrix.device }}-${{ matrix.interface }}.out From 0567d2ff4565516f7928b5eac6d1ae6861842eca Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 12 Jun 2026 08:24:34 -0400 Subject: [PATCH 2/6] ci: double the Frontier dependency-build step timeout to 120 minutes Four non-AMD Frontier jobs (CCE gpu-omp x2, gpu-acc, cpu) died uniformly at ~51-55 min in yesterday's run; login-node build contention makes the 60-minute budget too tight on bad days. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c53849e599..8dbbb3571a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -420,7 +420,7 @@ jobs: - name: Fetch Dependencies if: matrix.cluster != 'phoenix' - timeout-minutes: 60 + timeout-minutes: 120 run: bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }} - name: Build From b0c4d101ec9e4f53a3c099e1a838b8fd8d34d718 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 12 Jun 2026 08:37:31 -0400 Subject: [PATCH 3/6] ci: shard the Frontier AMD cpu test job It ran the full suite in one 1:59-walltime SLURM job (job 80982103050 died at the limit); the AMD gpu jobs were already sharded 2-way - the cpu job now matches. The job name gains the [i/2] suffix; if any branch-protection required-check pins the old unsharded name, update it. --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8dbbb3571a..2d0418564f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -402,6 +402,13 @@ jobs: cluster_name: 'Oak Ridge | Frontier (AMD)' device: 'cpu' interface: 'none' + shard: '1/2' + - runner: 'frontier' + cluster: 'frontier_amd' + cluster_name: 'Oak Ridge | Frontier (AMD)' + device: 'cpu' + interface: 'none' + shard: '2/2' runs-on: group: phoenix labels: ${{ matrix.runner }} From 32b22ad9a86b708c634a099b1eb90f35e1aeb52d Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 12 Jun 2026 08:57:51 -0400 Subject: [PATCH 4/6] ci: harden the prebuild shard validation and marker protocol Copilot review fixes: shard parts validated independently and the full i/N shape enforced (1/, /2, and bare 12 now rejected); shard 1 clears both markers at start so stale state from reruns cannot skip the wait; a failure marker written via ERR trap lets waiting shards fail fast instead of burning the 90-minute timeout. --- .github/scripts/prebuild-case-optimization.sh | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/scripts/prebuild-case-optimization.sh b/.github/scripts/prebuild-case-optimization.sh index a5869b7dfa..7fb9af0041 100755 --- a/.github/scripts/prebuild-case-optimization.sh +++ b/.github/scripts/prebuild-case-optimization.sh @@ -27,12 +27,26 @@ esac # case list. Unset = build all cases in one job (default; other clusters). shard="${job_shard:-}" if [ -n "$shard" ]; then + # Validate full shape: must be exactly "digits/digits" — one slash with + # non-empty, purely numeric, non-leading-zero parts on both sides. + # Split first, then validate each part independently so that inputs like + # "1/" "/2" "//" "1/2/3" "a/b" "12" are all caught before any arithmetic. shard_idx="${shard%%/*}" shard_count="${shard##*/}" - case "${shard_idx}${shard_count}" in - ''|*[!0-9]*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;; + # Reject if no slash (idx and count are equal and equal to the whole string) + case "$shard_idx" in + ''|*[!0-9]*|0*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;; esac - if [ "$shard" != "$shard_idx/$shard_count" ] || [ "$shard_idx" -lt 1 ] || [ "$shard_idx" -gt "$shard_count" ]; then + case "$shard_count" in + ''|*[!0-9]*|0*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;; + esac + # Confirm the string is exactly "idx/count" — catches "12" (no slash) and + # "1/2/3" (extra slash, where idx=1 and count=2/3 would have failed above, + # but this is an extra safety net). + if [ "$shard" != "$shard_idx/$shard_count" ]; then + echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 + fi + if [ "$shard_idx" -lt 1 ] || [ "$shard_idx" -gt "$shard_count" ]; then echo "ERROR: bad shard '$shard' (expected i/N with 1 <= i <= N)"; exit 1 fi fi @@ -61,22 +75,33 @@ esac # Case-optimized simulation builds land in per-case hash-named staging dirs, # but syscheck/pre_process/post_process hash identically across these cases. # Concurrent shards must not build those shared staging dirs simultaneously: -# shard 1 builds them first; the other shards wait for its marker, after -# which their builds no-op in the shared dirs. +# shard 1 builds them first and drops a done marker; other shards wait for it, +# after which their builds no-op in the shared dirs. if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then - shared_marker="build/.prebuild-shared-targets-done" + shared_marker_done="build/.prebuild-shared-targets-done" + shared_marker_failed="build/.prebuild-shared-targets-failed" set -- benchmarks/*/case.py first_case="$1" if [ "$shard_idx" -eq 1 ]; then + # Remove both markers at the start so reruns and manual invocations + # never observe stale state from a prior run. + rm -f "$shared_marker_done" "$shared_marker_failed" echo "=== Shard 1/$shard_count: building shared targets ===" + # Write the failure marker if the build exits non-zero so other shards + # can detect the failure immediately instead of waiting 90 minutes. + trap 'touch "$shared_marker_failed"' ERR ./mfc.sh build -i "$first_case" -t syscheck pre_process post_process --case-optimization $gpu_opts -j 8 - touch "$shared_marker" + trap - ERR + touch "$shared_marker_done" else echo "=== Shard $shard_idx/$shard_count: waiting for shard 1 to build shared targets ===" waited=0 - until [ -f "$shared_marker" ]; do + until [ -f "$shared_marker_done" ]; do + if [ -f "$shared_marker_failed" ]; then + echo "ERROR: shard 1 failed to build shared targets; see shard 1 log"; exit 1 + fi if [ "$waited" -ge 5400 ]; then - echo "ERROR: timed out waiting for $shared_marker"; exit 1 + echo "ERROR: timed out waiting for $shared_marker_done"; exit 1 fi sleep 30 waited=$((waited + 30)) From cf065ceab4e3da9192fda29185323752daf7486e Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 14 Jun 2026 12:40:18 -0400 Subject: [PATCH 5/6] ci: fail the benchmark job when the PR run fails, not just when its YAML is missing run_parallel_benchmarks.sh only hard-failed if the output YAML was absent; a non-zero PR SLURM exit (all cases SIGTERM'd/crashed/hung) was downgraded to a warning, so a broken PR benchmark passed green as long as a partial YAML existed (observed on #1588, where phoenix-bench node contention SIGTERM'd every PR case yet the check stayed green). Now a genuine PR-job failure exits 1. pr_exit is reliable: run_monitored_slurm_job.sh re-checks sacct and returns 0 when the job actually COMPLETED 0:0, so this does not red-cross on monitor flakiness. Scoped to PR only -- a master/baseline infra failure stays a warning. --- .github/scripts/run_parallel_benchmarks.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/scripts/run_parallel_benchmarks.sh b/.github/scripts/run_parallel_benchmarks.sh index 15609d3c2b..7d5a410633 100755 --- a/.github/scripts/run_parallel_benchmarks.sh +++ b/.github/scripts/run_parallel_benchmarks.sh @@ -62,6 +62,12 @@ bash "${SCRIPT_DIR}/run_monitored_slurm_job.sh" "$pr_job_id" "pr/${job_slug}.out if [ "$pr_exit" -ne 0 ]; then echo "PR job exited with code: $pr_exit" tail -n 50 "pr/${job_slug}.out" 2>/dev/null || echo " Could not read PR log" + # The PR benchmark run genuinely failed (cases crashed/hung/SIGTERM'd, not a + # monitor false-positive -- run_monitored_slurm_job.sh re-checks sacct). Fail + # the job instead of falling through to the YAML-exists check, which would let + # a broken PR pass green as long as a partial YAML was written. Scoped to PR + # only: a master/baseline infra flake stays a warning and does not red-cross. + exit 1 else echo "PR job completed successfully" fi From 8fbac1591e0548117af629ddce8148c83a40805d Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sun, 14 Jun 2026 15:15:32 -0400 Subject: [PATCH 6/6] benchmarks/igr: use hardcoded Taylor-Green patch (hcid 380) so one binary serves all bench cases --- benchmarks/igr/case.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarks/igr/case.py b/benchmarks/igr/case.py index 423d7e1a41..d54e3bbdaf 100644 --- a/benchmarks/igr/case.py +++ b/benchmarks/igr/case.py @@ -102,10 +102,11 @@ "patch_icpp(1)%length_x": 2 * math.pi * L, "patch_icpp(1)%length_y": 2 * math.pi * L, "patch_icpp(1)%length_z": 2 * math.pi * L, - "patch_icpp(1)%vel(1)": f"{V0}*sin(x/{L})*cos(y/{L})*sin(z/{L})", - "patch_icpp(1)%vel(2)": f"-{V0}*cos(x/{L})*sin(y/{L})*sin(z/{L})", + "patch_icpp(1)%vel(1)": 0.0, + "patch_icpp(1)%vel(2)": 0.0, "patch_icpp(1)%vel(3)": 0, - "patch_icpp(1)%pres": f"{P0} + ({rho0}*{V0}**2/16)*(cos(2*x/{L}) + cos(2*y/{L}))*(cos(2*z/{L}) + 2)", + "patch_icpp(1)%pres": 0.0, + "patch_icpp(1)%hcid": 380, "patch_icpp(1)%alpha_rho(1)": 1, "patch_icpp(1)%alpha(1)": 1, # Fluids Physical Parameters