From cf8040f9174d1812ed4039580d4074e217dc8aeb Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 11 Sep 2026 10:38:32 -0400 Subject: [PATCH 1/3] Make the IB force-reduction receive buffers device resident `s_communicate_ib_forces` passed `recv_forces_snap`, `recv_torques_snap`, `recv_ids` and `recv_ft` into GPU kernels through `copy`/`copyin`, but allocated them with a plain `allocate`, so they never entered the device present table. On Cray OpenMP offload this aborts at the first time step of any moving-IB case run on more than one rank: ACC: find_in_present_table failed for 'recv_forces_snap(:,:)' from m_ibm.fpp:1338 ACC: libcrayacc/acc_runtime.c:703 CRAY_ACC_ERROR - Variable not found in present table The sibling send buffers `send_ids`/`send_ft` are already `@:ALLOCATE`d and pushed with `GPU_UPDATE`, so the receive side was simply inconsistent with them. Allocate the four receive arrays the same way and update them to the device after the host writes (the zeroing before each accumulation pass, and each `MPI_UNPACK`), which also lets the kernels drop the `copy`/`copyin` of those arrays. Reproduced on Frontier (`./mfc.sh build --gpu mp`, cpe/25.03, rocm/6.3.1) with a 2D moving flat plate on 4 ranks; the same case runs to completion with this change, and an OpenACC build was unaffected either way. Fixes #1840 Claude-Session: https://claude.ai/code/session_01HMJ7cycfo7kTFSFq5yhHLG --- src/simulation/m_ibm.fpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/simulation/m_ibm.fpp b/src/simulation/m_ibm.fpp index 99a212260..ac09cee94 100644 --- a/src/simulation/m_ibm.fpp +++ b/src/simulation/m_ibm.fpp @@ -97,8 +97,8 @@ contains #ifdef MFC_MPI if (num_procs > 1) then @:ALLOCATE(send_ids(size(patch_ib)), send_ft(6, size(patch_ib))) - allocate (recv_forces_snap(size(patch_ib), 3), recv_torques_snap(size(patch_ib), 3), recv_ids(size(patch_ib)), & - & recv_ft(6, size(patch_ib))) + @:ALLOCATE(recv_forces_snap(size(patch_ib), 3), recv_torques_snap(size(patch_ib), 3), recv_ids(size(patch_ib)), & + & recv_ft(6, size(patch_ib))) end if #endif @@ -1310,6 +1310,7 @@ contains recv_forces_snap = 0._wp recv_torques_snap = 0._wp + $:GPU_UPDATE(device='[recv_forces_snap, recv_torques_snap]') tag = 300 do k = 1, min(2*ib_neighborhood_radius, num_procs_${X}$ - 1) @@ -1335,8 +1336,8 @@ contains call MPI_UNPACK(ib_force_recv_buf, buf_size, unpack_pos, recv_ids, recv_count, MPI_INTEGER, & & MPI_COMM_WORLD, ierr) call MPI_UNPACK(ib_force_recv_buf, buf_size, unpack_pos, recv_ft, 6*recv_count, mpi_p, MPI_COMM_WORLD, ierr) - $:GPU_PARALLEL_LOOP(private='[i, j]', copyin='[recv_ft, recv_ids]', copy='[forces, torques, & - & recv_forces_snap, recv_torques_snap]') + $:GPU_UPDATE(device='[recv_ids, recv_ft]') + $:GPU_PARALLEL_LOOP(private='[i, j]', copy='[forces, torques]') do i = 1, recv_count call s_get_neighborhood_idx(recv_ids(i), j) if (j > 0) then @@ -1381,7 +1382,8 @@ contains call MPI_UNPACK(ib_force_recv_buf, buf_size, unpack_pos, recv_ids, recv_count, MPI_INTEGER, & & MPI_COMM_WORLD, ierr) call MPI_UNPACK(ib_force_recv_buf, buf_size, unpack_pos, recv_ft, 6*recv_count, mpi_p, MPI_COMM_WORLD, ierr) - $:GPU_PARALLEL_LOOP(private='[i, j]', copyin='[recv_ft, recv_ids]', copy='[forces, torques]') + $:GPU_UPDATE(device='[recv_ids, recv_ft]') + $:GPU_PARALLEL_LOOP(private='[i, j]', copy='[forces, torques]') do i = 1, recv_count call s_get_neighborhood_idx(recv_ids(i), j) if (j > 0) then @@ -1599,7 +1601,7 @@ contains #ifdef MFC_MPI if (num_procs > 1) then @:DEALLOCATE(send_ids, send_ft) - deallocate (recv_forces_snap, recv_torques_snap, recv_ids, recv_ft) + @:DEALLOCATE(recv_forces_snap, recv_torques_snap, recv_ids, recv_ft) end if #endif From b2ef85321d719c5dc794fc4804ad4bfa397c7d23 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 11 Sep 2026 22:46:06 -0500 Subject: [PATCH 2/3] Fix three doc-reference lint hits on master --- docs/documentation/contributing.md | 2 +- docs/documentation/gpuParallelization.md | 2 +- docs/documentation/testing.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/documentation/contributing.md b/docs/documentation/contributing.md index d42dc09e8..dd3c0f33c 100644 --- a/docs/documentation/contributing.md +++ b/docs/documentation/contributing.md @@ -217,7 +217,7 @@ Both human reviewers and AI code reviewers reference this section. ### Parameter Plumbing -- **Derived-type parameters are not auto-broadcast.** `generated_bcast.fpp` covers namelist *scalars* only. Each derived type (`chem_params`, `lag_params`, `rburn`) needs a hand-written `_emit_` in `toolchain/mfc/params/generators/fortran_gen.py` plus its call site in that generator's simulation branch, and, if it is read on device, an explicit `$:GPU_UPDATE(device='[name]')` in both the target's `m_global_parameters.fpp` and `src/simulation/m_start_up.fpp` — `GPU_DECLARE` alone does not make it device-resident. Regrouping existing scalars into a derived type silently drops their broadcast, leaving every non-root rank holding the `dflt_real` sentinel. Single-rank golden files cannot catch this, so pair such a change with a `ppn=2` test and confirm it fails without the emitter. +- **Derived-type parameters are not auto-broadcast.** `generated_bcast.fpp` covers namelist *scalars* only. Each derived type (`chem_params`, `lag_params`, `rburn`) needs a hand-written `_emit_` in `toolchain/mfc/params/generators/fortran_gen.py` plus its call site in that generator's simulation branch, and, if it is read on device, an explicit ``$:GPU_UPDATE(device='[name]')`` in both the target's `m_global_parameters.fpp` and `src/simulation/m_start_up.fpp` — `GPU_DECLARE` alone does not make it device-resident. Regrouping existing scalars into a derived type silently drops their broadcast, leaving every non-root rank holding the `dflt_real` sentinel. Single-rank golden files cannot catch this, so pair such a change with a `ppn=2` test and confirm it fails without the emitter. - **A `patch_ib` member that immersed-boundary ghost-point code reads must also be set in `s_add_cloud_particle`** (`src/simulation/m_particle_cloud.fpp`). `particle_cloud_ibs` is allocated without default initialization, and `s_reduce_ib_patch_array` copies the whole struct into `patch_ib`, overwriting the defaults assigned in `s_assign_default_values_to_user_inputs`. Anything left unset reaches the solver as uninitialized memory, and only where the allocation is not already zero-filled. A platform-only NaN is the signature of this class: a garbage `v_blow` once failed an AMD lane with `ICFL is NaN` while every NVIDIA lane and all local runs passed. - **Runtime checks go where they run.** Shared constraints belong in `src/common/m_checker_common.fpp`, simulation-only ones in `src/simulation/m_checker.fpp`, and pre- and post-process ones in their own `m_checker.fpp`. Those two `s_check_inputs` are currently empty; that is still the correct home for their checks, not `m_checker_common`. - **Analytic initial conditions are compiled into the binary** and their expressions are AST-validated at case load, so syntax errors and unknown variables surface immediately and by name. Each IC variable maps to an `eqn_idx` expression in `QPVF_IDX_VARS` (`toolchain/mfc/case.py`); adding a patch-settable conserved variable means updating that map and the Fortran `eqn_idx` builder together, because a mismatch is a silent wrong index. diff --git a/docs/documentation/gpuParallelization.md b/docs/documentation/gpuParallelization.md index 8b205715d..b559c17fe 100644 --- a/docs/documentation/gpuParallelization.md +++ b/docs/documentation/gpuParallelization.md @@ -874,7 +874,7 @@ answer is wrong, or one backend diverges from all the others. always use `GPU_PARALLEL_LOOP` / `END_GPU_PARALLEL_LOOP`. - **An array whose bound is a device global** (`dimension(num_fluids)`, `dimension(num_species)`) may be passed to a device routine from a parallel-loop body, - but **not from inside another `GPU_ROUTINE(parallelism='[seq]')`**. Cray OpenACC rejects + but **not from inside another ``GPU_ROUTINE(parallelism='[seq]')``**. Cray OpenACC rejects the second form with `ftn-7066 ... Global in accelerator routine without declare`, and reports it at whatever line it gave up on: remove one trigger and the message walks forward to the next call, so the reported line is not the cause. Only the plain lanes diff --git a/docs/documentation/testing.md b/docs/documentation/testing.md index be50883f7..b23e15231 100644 --- a/docs/documentation/testing.md +++ b/docs/documentation/testing.md @@ -92,7 +92,7 @@ If a trace is empty (that is, the empty string `""`), it will not appear in the Finally, the case is appended to the `cases` list, which will be returned by the `list_cases` function. -### Selection and Execution Pitfalls +### Selection and Execution Pitfalls {#selection-and-execution-pitfalls} Each of these fails quietly rather than loudly. From 29034b0e0c950157116cd708b796dba40df4dd20 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 11 Sep 2026 22:58:16 -0500 Subject: [PATCH 3/3] Update only the received entries of the IB force buffers --- src/simulation/m_ibm.fpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/m_ibm.fpp b/src/simulation/m_ibm.fpp index ac09cee94..af4dbc380 100644 --- a/src/simulation/m_ibm.fpp +++ b/src/simulation/m_ibm.fpp @@ -1336,7 +1336,7 @@ contains call MPI_UNPACK(ib_force_recv_buf, buf_size, unpack_pos, recv_ids, recv_count, MPI_INTEGER, & & MPI_COMM_WORLD, ierr) call MPI_UNPACK(ib_force_recv_buf, buf_size, unpack_pos, recv_ft, 6*recv_count, mpi_p, MPI_COMM_WORLD, ierr) - $:GPU_UPDATE(device='[recv_ids, recv_ft]') + $:GPU_UPDATE(device='[recv_ids(1:recv_count), recv_ft(:, 1:recv_count)]') $:GPU_PARALLEL_LOOP(private='[i, j]', copy='[forces, torques]') do i = 1, recv_count call s_get_neighborhood_idx(recv_ids(i), j) @@ -1382,7 +1382,7 @@ contains call MPI_UNPACK(ib_force_recv_buf, buf_size, unpack_pos, recv_ids, recv_count, MPI_INTEGER, & & MPI_COMM_WORLD, ierr) call MPI_UNPACK(ib_force_recv_buf, buf_size, unpack_pos, recv_ft, 6*recv_count, mpi_p, MPI_COMM_WORLD, ierr) - $:GPU_UPDATE(device='[recv_ids, recv_ft]') + $:GPU_UPDATE(device='[recv_ids(1:recv_count), recv_ft(:, 1:recv_count)]') $:GPU_PARALLEL_LOOP(private='[i, j]', copy='[forces, torques]') do i = 1, recv_count call s_get_neighborhood_idx(recv_ids(i), j)