From 26572fb381972b129e83a648dcc9dd241170de5a Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 1 Apr 2026 18:28:36 -0400 Subject: [PATCH 1/4] ci: use single-CUDA Docker tags for NVHPC to fit runner disk --- .github/workflows/test.yml | 64 ++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 00eeb3c5f8..41b63df709 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -179,42 +179,44 @@ jobs: # NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp) # Every release from 23.11 through 26.3 (current) - - { nvhpc: '23.11', target: cpu } - - { nvhpc: '23.11', target: gpu } - - { nvhpc: '24.1', target: cpu } - - { nvhpc: '24.1', target: gpu } - - { nvhpc: '24.3', target: cpu } - - { nvhpc: '24.3', target: gpu } - - { nvhpc: '24.5', target: cpu } - - { nvhpc: '24.5', target: gpu } - - { nvhpc: '24.7', target: cpu } - - { nvhpc: '24.7', target: gpu } - - { nvhpc: '24.9', target: cpu } - - { nvhpc: '24.9', target: gpu } - - { nvhpc: '24.11', target: cpu } - - { nvhpc: '24.11', target: gpu } - - { nvhpc: '25.1', target: cpu } - - { nvhpc: '25.1', target: gpu } - - { nvhpc: '25.3', target: cpu } - - { nvhpc: '25.3', target: gpu } - - { nvhpc: '25.5', target: cpu } - - { nvhpc: '25.5', target: gpu } - - { nvhpc: '25.7', target: cpu } - - { nvhpc: '25.7', target: gpu } - - { nvhpc: '25.9', target: cpu } - - { nvhpc: '25.9', target: gpu } - - { nvhpc: '25.11', target: cpu } - - { nvhpc: '25.11', target: gpu } - - { nvhpc: '26.1', target: cpu } - - { nvhpc: '26.1', target: gpu } - - { nvhpc: '26.3', target: cpu } - - { nvhpc: '26.3', target: gpu } + # Using single-CUDA tags instead of cuda_multi to fit GitHub runner disk. + # nvhpc -> highest bundled CUDA 12.x (from release notes) + - { nvhpc: '23.11', target: cpu, cuda: '12.3' } + - { nvhpc: '23.11', target: gpu, cuda: '12.3' } + - { nvhpc: '24.1', target: cpu, cuda: '12.3' } + - { nvhpc: '24.1', target: gpu, cuda: '12.3' } + - { nvhpc: '24.3', target: cpu, cuda: '12.3' } + - { nvhpc: '24.3', target: gpu, cuda: '12.3' } + - { nvhpc: '24.5', target: cpu, cuda: '12.4' } + - { nvhpc: '24.5', target: gpu, cuda: '12.4' } + - { nvhpc: '24.7', target: cpu, cuda: '12.5' } + - { nvhpc: '24.7', target: gpu, cuda: '12.5' } + - { nvhpc: '24.9', target: cpu, cuda: '12.6' } + - { nvhpc: '24.9', target: gpu, cuda: '12.6' } + - { nvhpc: '24.11', target: cpu, cuda: '12.6' } + - { nvhpc: '24.11', target: gpu, cuda: '12.6' } + - { nvhpc: '25.1', target: cpu, cuda: '12.6' } + - { nvhpc: '25.1', target: gpu, cuda: '12.6' } + - { nvhpc: '25.3', target: cpu, cuda: '12.8' } + - { nvhpc: '25.3', target: gpu, cuda: '12.8' } + - { nvhpc: '25.5', target: cpu, cuda: '12.8' } + - { nvhpc: '25.5', target: gpu, cuda: '12.8' } + - { nvhpc: '25.7', target: cpu, cuda: '12.8' } + - { nvhpc: '25.7', target: gpu, cuda: '12.8' } + - { nvhpc: '25.9', target: cpu, cuda: '12.9' } + - { nvhpc: '25.9', target: gpu, cuda: '12.9' } + - { nvhpc: '25.11', target: cpu, cuda: '12.9' } + - { nvhpc: '25.11', target: gpu, cuda: '12.9' } + - { nvhpc: '26.1', target: cpu, cuda: '12.9' } + - { nvhpc: '26.1', target: gpu, cuda: '12.9' } + - { nvhpc: '26.3', target: cpu, cuda: '12.9' } + - { nvhpc: '26.3', target: gpu, cuda: '12.9' } fail-fast: false continue-on-error: true runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }} container: - image: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }} + image: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda{1}-ubuntu22.04', matrix.nvhpc, matrix.cuda) || '' }} options: ${{ matrix.nvhpc && '--security-opt seccomp=unconfined' || '' }} env: CC: ${{ matrix.nvhpc && 'nvc' || '' }} From d6339ae8aae59d47e4e97b2012df278022da779f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 1 Apr 2026 18:37:24 -0400 Subject: [PATCH 2/4] Fix comment formatting in CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index da1c6d035d..b806a7b4c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ cmake_minimum_required(VERSION 3.20) -# We include C as a language because - for some reason - +# We include C as a language because - for some reason # FIND_LIBRARY_USE_LIB64_PATHS is otherwise ignored. project(MFC LANGUAGES C CXX Fortran) From 3b651b10d92e811c80c5cd076196e088ad5bdd0a Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 1 Apr 2026 20:54:48 -0400 Subject: [PATCH 3/4] Refactor NVHPC setup in GitHub Actions workflow Updated NVHPC configuration to simplify CUDA handling and improve disk management during CI runs. --- .github/workflows/test.yml | 204 +++++++++++++++++++++++-------------- 1 file changed, 125 insertions(+), 79 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41b63df709..aea7187d3b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -179,61 +179,59 @@ jobs: # NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp) # Every release from 23.11 through 26.3 (current) - # Using single-CUDA tags instead of cuda_multi to fit GitHub runner disk. - # nvhpc -> highest bundled CUDA 12.x (from release notes) - - { nvhpc: '23.11', target: cpu, cuda: '12.3' } - - { nvhpc: '23.11', target: gpu, cuda: '12.3' } - - { nvhpc: '24.1', target: cpu, cuda: '12.3' } - - { nvhpc: '24.1', target: gpu, cuda: '12.3' } - - { nvhpc: '24.3', target: cpu, cuda: '12.3' } - - { nvhpc: '24.3', target: gpu, cuda: '12.3' } - - { nvhpc: '24.5', target: cpu, cuda: '12.4' } - - { nvhpc: '24.5', target: gpu, cuda: '12.4' } - - { nvhpc: '24.7', target: cpu, cuda: '12.5' } - - { nvhpc: '24.7', target: gpu, cuda: '12.5' } - - { nvhpc: '24.9', target: cpu, cuda: '12.6' } - - { nvhpc: '24.9', target: gpu, cuda: '12.6' } - - { nvhpc: '24.11', target: cpu, cuda: '12.6' } - - { nvhpc: '24.11', target: gpu, cuda: '12.6' } - - { nvhpc: '25.1', target: cpu, cuda: '12.6' } - - { nvhpc: '25.1', target: gpu, cuda: '12.6' } - - { nvhpc: '25.3', target: cpu, cuda: '12.8' } - - { nvhpc: '25.3', target: gpu, cuda: '12.8' } - - { nvhpc: '25.5', target: cpu, cuda: '12.8' } - - { nvhpc: '25.5', target: gpu, cuda: '12.8' } - - { nvhpc: '25.7', target: cpu, cuda: '12.8' } - - { nvhpc: '25.7', target: gpu, cuda: '12.8' } - - { nvhpc: '25.9', target: cpu, cuda: '12.9' } - - { nvhpc: '25.9', target: gpu, cuda: '12.9' } - - { nvhpc: '25.11', target: cpu, cuda: '12.9' } - - { nvhpc: '25.11', target: gpu, cuda: '12.9' } - - { nvhpc: '26.1', target: cpu, cuda: '12.9' } - - { nvhpc: '26.1', target: gpu, cuda: '12.9' } - - { nvhpc: '26.3', target: cpu, cuda: '12.9' } - - { nvhpc: '26.3', target: gpu, cuda: '12.9' } + - { nvhpc: '23.11', target: cpu } + - { nvhpc: '23.11', target: gpu } + - { nvhpc: '24.1', target: cpu } + - { nvhpc: '24.1', target: gpu } + - { nvhpc: '24.3', target: cpu } + - { nvhpc: '24.3', target: gpu } + - { nvhpc: '24.5', target: cpu } + - { nvhpc: '24.5', target: gpu } + - { nvhpc: '24.7', target: cpu } + - { nvhpc: '24.7', target: gpu } + - { nvhpc: '24.9', target: cpu } + - { nvhpc: '24.9', target: gpu } + - { nvhpc: '24.11', target: cpu } + - { nvhpc: '24.11', target: gpu } + - { nvhpc: '25.1', target: cpu } + - { nvhpc: '25.1', target: gpu } + - { nvhpc: '25.3', target: cpu } + - { nvhpc: '25.3', target: gpu } + - { nvhpc: '25.5', target: cpu } + - { nvhpc: '25.5', target: gpu } + - { nvhpc: '25.7', target: cpu } + - { nvhpc: '25.7', target: gpu } + - { nvhpc: '25.9', target: cpu } + - { nvhpc: '25.9', target: gpu } + - { nvhpc: '25.11', target: cpu } + - { nvhpc: '25.11', target: gpu } + - { nvhpc: '26.1', target: cpu } + - { nvhpc: '26.1', target: gpu } + - { nvhpc: '26.3', target: cpu } + - { nvhpc: '26.3', target: gpu } fail-fast: false continue-on-error: true runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }} - container: - image: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda{1}-ubuntu22.04', matrix.nvhpc, matrix.cuda) || '' }} - options: ${{ matrix.nvhpc && '--security-opt seccomp=unconfined' || '' }} env: - CC: ${{ matrix.nvhpc && 'nvc' || '' }} - CXX: ${{ matrix.nvhpc && 'nvc++' || '' }} - FC: ${{ matrix.nvhpc && 'nvfortran' || '' }} - OMPI_ALLOW_RUN_AS_ROOT: ${{ matrix.nvhpc && '1' || '' }} - OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: ${{ matrix.nvhpc && '1' || '' }} - PMIX_MCA_gds: ${{ matrix.nvhpc && 'hash' || '' }} - OMPI_MCA_hwloc_base_binding_policy: ${{ matrix.nvhpc && 'none' || '' }} - FFLAGS: ${{ matrix.nvhpc && '-tp=px -Kieee -noswitcherror' || '' }} - CFLAGS: ${{ matrix.nvhpc && '-tp=px' || '' }} - CXXFLAGS: ${{ matrix.nvhpc && '-tp=px' || '' }} + # Image tag for NVHPC jobs; empty for non-NVHPC jobs. + NVHPC_IMAGE: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }} steps: - - name: Git safe directory + # ── NVHPC: free disk before pulling the ~25-30 GB cuda_multi image ── + - name: Free disk space if: matrix.nvhpc - run: git config --global --add safe.directory /__w/MFC/MFC + run: | + echo "=== Disk before cleanup ===" + df -h / + sudo rm -rf /usr/share/dotnet /usr/local/lib/android \ + /opt/ghc /usr/local/share/boost /opt/hostedtoolcache \ + /usr/local/graalvm /usr/local/.ghcup \ + /usr/local/share/chromium /usr/local/lib/node_modules + sudo docker image prune -af + sudo apt-get clean + echo "=== Disk after cleanup ===" + df -h / - name: Clone uses: actions/checkout@v4 @@ -276,6 +274,62 @@ jobs: echo "Coverage cache: none available — full test suite will run" fi + # ── NVHPC: pull image and start a long-lived container ────────────── + # Replaces the container: directive so we can free disk space first. + # Uses "docker run -d ... sleep infinity" + "docker exec" to preserve + # installed packages and env vars across steps. + - name: Pull NVHPC container + if: matrix.nvhpc + run: docker pull "$NVHPC_IMAGE" + + - name: Start NVHPC container + if: matrix.nvhpc + run: | + docker run -d --name nvhpc \ + --security-opt seccomp=unconfined \ + -v "${{ github.workspace }}:/workspace" \ + -w /workspace \ + -e CC=nvc \ + -e CXX=nvc++ \ + -e FC=nvfortran \ + -e OMPI_ALLOW_RUN_AS_ROOT=1 \ + -e OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \ + -e PMIX_MCA_gds=hash \ + -e OMPI_MCA_hwloc_base_binding_policy=none \ + -e "FFLAGS=-tp=px -Kieee -noswitcherror" \ + -e CFLAGS=-tp=px \ + -e CXXFLAGS=-tp=px \ + "$NVHPC_IMAGE" sleep infinity + + - name: Setup NVHPC + if: matrix.nvhpc + run: | + docker exec nvhpc bash -c ' + set -e + apt-get update -y + apt-get install -y cmake python3 python3-venv python3-pip \ + libfftw3-dev libhdf5-dev hdf5-tools git + + # Set up NVHPC HPC-X MPI runtime paths + HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../.. + MPI_LIB=$(mpifort --showme:link | grep -oP "(?<=-L)\S+" | head -1) + + # Persist env vars for subsequent docker exec calls + cat > /etc/nvhpc-env.sh <> $GITHUB_ENV echo "MPICXX=mpiicpx" >> $GITHUB_ENV - # --- NVHPC container setup --- - - name: Setup NVHPC - if: matrix.nvhpc - run: | - apt-get update -y - apt-get install -y cmake python3 python3-venv python3-pip \ - libfftw3-dev libhdf5-dev hdf5-tools git - # Set up NVHPC HPC-X MPI runtime paths - HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../.. - MPI_LIB=$(mpifort --showme:link | grep -oP '(?<=-L)\S+' | head -1) - echo "LD_LIBRARY_PATH=${MPI_LIB}:${HPCX_DIR}/ucx/lib:${HPCX_DIR}/ucc/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - # Container MPI fixes: PMIx shared-memory, hwloc binding - echo "PMIX_MCA_gds=hash" >> $GITHUB_ENV - echo "OMPI_MCA_hwloc_base_binding_policy=none" >> $GITHUB_ENV - echo "OMPI_MCA_rmaps_base_oversubscribe=1" >> $GITHUB_ENV - # Debug: confirm compiler flags are set - echo "=== NVHPC Environment ===" - echo "FFLAGS=$FFLAGS" - echo "CFLAGS=$CFLAGS" - echo "CXXFLAGS=$CXXFLAGS" - nvfortran --version - cat /proc/cpuinfo | grep "model name" | head -1 - - # --- Standard build + test --- + # ── Standard build + test ─────────────────────────────────────────── - name: Build if: '!matrix.nvhpc' run: | @@ -356,22 +387,37 @@ jobs: TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }} ONLY_CHANGES: ${{ github.event_name == 'pull_request' && '--only-changes' || '' }} - # --- NVHPC build + test --- + # ── NVHPC build + test (via docker exec into long-lived container) ── - name: Build (NVHPC) if: matrix.nvhpc && matrix.target == 'cpu' - run: /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all + run: | + docker exec nvhpc bash -c ' + source /etc/nvhpc-env.sh + /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all + ' - name: Build (NVHPC GPU) if: matrix.nvhpc && matrix.target == 'gpu' - run: | - /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu acc - /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp + run: | + docker exec nvhpc bash -c ' + source /etc/nvhpc-env.sh + /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu acc + /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp + ' - name: Test (NVHPC) if: matrix.nvhpc && matrix.target == 'cpu' - run: | - ulimit -s unlimited || ulimit -s 65536 || true - /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all + run: | + docker exec nvhpc bash -c ' + source /etc/nvhpc-env.sh + ulimit -s unlimited || ulimit -s 65536 || true + /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all + ' + + # ── Cleanup ───────────────────────────────────────────────────────── + - name: Stop NVHPC container + if: always() && matrix.nvhpc + run: docker rm -f nvhpc || true self: name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }}${{ matrix.shard != '' && format(' [{0}]', matrix.shard) || '' }})" From ae9462d9f2140dfccc526869cd6017619de2f5bb Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 2 Apr 2026 08:12:01 -0400 Subject: [PATCH 4/4] ci: fix git 'dubious ownership' spam in NVHPC Docker container The repo is bind-mounted into the container from the host runner, so git sees a different owner and emits 'fatal: detected dubious ownership' on every git command. This causes the test runner to dump the git diff help text 546 times (once per test), bloating the CI log to 80,000+ lines. Fix: git config --global --add safe.directory /workspace in Setup NVHPC. --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aea7187d3b..88b813ed7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -310,6 +310,11 @@ jobs: apt-get install -y cmake python3 python3-venv python3-pip \ libfftw3-dev libhdf5-dev hdf5-tools git + # The repo is bind-mounted from the host so git sees a different + # owner. Mark it safe to suppress "dubious ownership" errors that + # otherwise spam 80 000+ lines into the CI log. + git config --global --add safe.directory /workspace + # Set up NVHPC HPC-X MPI runtime paths HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../.. MPI_LIB=$(mpifort --showme:link | grep -oP "(?<=-L)\S+" | head -1)