ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-and-test:
    name: Build & Test (GPU)
    # Never execute untrusted fork code on the persistent self-hosted GPU runner.
    # Branch PRs within this repository are gated; fork PRs retain CPU-only checks.
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    # Requires a self-hosted runner with CUDA, GPU, and the label 'gpu'
    runs-on: [self-hosted, gpu, linux]

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Load CUDA module (no Spack)
        shell: bash
        run: |
          export PATH=$(echo "$PATH" | tr ':' '\n' | grep -v '/spack/' | paste -sd:)
          if [[ -f /etc/profile.d/lmod.sh ]]; then
            source /etc/profile.d/lmod.sh
          else
            echo "lmod not found; skipping module setup"
          fi
          if command -v module >/dev/null 2>&1; then
            module purge
            module load nvhpc/23.11/nvhpc-hpcx-cuda12
          else
            echo "module command not available; using existing CUDA on PATH"
          fi
          echo "PATH=$PATH"
          if ! command -v nvcc >/dev/null 2>&1; then
            echo "nvcc not found on PATH"
            exit 1
          fi
          nvcc --version
          nvidia-smi
          echo "PATH=$PATH" >> "$GITHUB_ENV"
          [[ -n "$LD_LIBRARY_PATH" ]] && echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$GITHUB_ENV" || true

      - name: Configure (Release)
        run: |
          if [[ -f /etc/profile.d/lmod.sh ]]; then
            source /etc/profile.d/lmod.sh
          fi
          if command -v module >/dev/null 2>&1; then
            module purge
            module load nvhpc/23.11/nvhpc-hpcx-cuda12
          fi
          cmake -S . -B build/ci \
            -DCMAKE_BUILD_TYPE=Release \
            -DBUILD_TESTING=ON \
            -DBUILD_EXAMPLES=ON \
            -DCMAKE_CUDA_ARCHITECTURES=native \
            -DCMAKE_INSTALL_PREFIX=./build/ci/install

      - name: Build
        shell: bash
        run: |
          if [[ -f /etc/profile.d/lmod.sh ]]; then source /etc/profile.d/lmod.sh; fi
          if command -v module >/dev/null 2>&1; then
            module purge
            module load nvhpc/23.11/nvhpc-hpcx-cuda12
          fi
          # nvc++ emits large LLVM intermediates. An unbounded parallel build can
          # exhaust the runner's small root filesystem through /tmp, leaving
          # truncated .ll files and misleading follow-on LLVM parse errors.
          compiler_tmp=$(mktemp -d "${RUNNER_TEMP:?}/fzgmod-nvcxx.XXXXXX")
          trap 'rm -rf -- "$compiler_tmp"' EXIT
          export TMPDIR="$compiler_tmp"
          df -h /tmp "$TMPDIR"
          cmake --build build/ci --parallel 4

      - name: Run tests
        shell: bash
        working-directory: build/ci
        run: |
          if [[ -f /etc/profile.d/lmod.sh ]]; then source /etc/profile.d/lmod.sh; fi
          if command -v module >/dev/null 2>&1; then
            module purge
            module load nvhpc/23.11/nvhpc-hpcx-cuda12
          fi
          ctest --output-on-failure

      - name: Upload test results
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: test-results
          path: build/ci/Testing/

  # ── ASan + UBSan ────────────────────────────────────────────────────────────
  sanitizers:
    name: Sanitizers (ASan + UBSan)
    # Keep the complete sanitizer pass on main/manual runs; pull requests get
    # the full Release GPU suite above without doubling scarce runner time.
    if: github.event_name != 'pull_request'
    runs-on: [self-hosted, gpu, linux]
    needs: build-and-test

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Configure (ASan preset)
        run: |
          if [[ -f /etc/profile.d/lmod.sh ]]; then
            source /etc/profile.d/lmod.sh
          fi
          if command -v module >/dev/null 2>&1; then
            module purge
            module load nvhpc/23.11/nvhpc-hpcx-cuda12
          fi
          # This runs on a self-hosted runner whose workspace persists across
          # jobs, so a stale build/asan/CMakeCache.txt from a prior run can
          # pin CMAKE_CUDA_ARCHITECTURES to a value baked in before it existed
          # in CMakeLists.txt (e.g. an autodetected sm_52, which predates the
          # block-scoped atomics RARE/RAZE/CLOG/HCLOG need) — wipe it so every
          # run reconfigures clean.
          rm -rf build/asan
          # nvc++ does not support -fsanitize; use GCC for the sanitizer build
          CC=gcc CXX=g++ cmake --preset asan

      - name: Build
        shell: bash
        run: |
          compiler_tmp=$(mktemp -d "${RUNNER_TEMP:?}/fzgmod-asan.XXXXXX")
          trap 'rm -rf -- "$compiler_tmp"' EXIT
          export TMPDIR="$compiler_tmp"
          df -h /tmp "$TMPDIR"
          cmake --build --preset asan --parallel 4

      - name: Run tests (ASan)
        run: |
          if [[ -f /etc/profile.d/lmod.sh ]]; then
            source /etc/profile.d/lmod.sh
          fi
          if command -v module >/dev/null 2>&1; then
            module purge
            module load nvhpc/23.11/nvhpc-hpcx-cuda12
          fi
          # ASan is baked into the shared libraries. The dynamic linker does not
          # guarantee it initializes before main(), so we must preload it first.
          # Without this, every test fails with:
          #   "ASan runtime does not come first in initial library list"
          ASAN_LIB=$(gcc -print-file-name=libasan.so)
          echo "Preloading: $ASAN_LIB"
          
          # CUDA memory pools conflict with ASan's huge shadow memory mapping (OOM error).
          # protect_shadow_gap=0 fixes this on Linux.
          export ASAN_OPTIONS="protect_shadow_gap=0"
          
          # Ignore known memory leaks in the NVIDIA driver
          echo -e "leak:libcuda\nleak:libnv" > lsan.supp
          export LSAN_OPTIONS="suppressions=$(pwd)/lsan.supp"
          
          LD_PRELOAD="$ASAN_LIB" ctest --preset asan --output-on-failure