gpu-validation.yml

name: Cross-Architecture Validation (EC2)

on:
  push:
    branches: [master, main]
    paths-ignore:
      - '**.md'
      - 'docs/**'
      - 'LICENSE'
      - '.github/workflows/version-release.yml'
  workflow_dispatch:
    inputs:
      ec2-instance-type:
        description: 'EC2 instance type'
        required: false
        default: 'g4dn.4xlarge'
        type: choice
        options:
          - g4dn.4xlarge
          - g4dn.2xlarge
          - g4dn.xlarge
  pull_request:
    types: [labeled]

# Prevent multiple EC2 instances from running concurrently.
# cancel-in-progress is false to avoid orphaning instances (stop-runner
# must complete for each run).
concurrency:
  group: ec2-validation
  cancel-in-progress: false

jobs:
  start-runner:
    name: Start EC2 runner
    if: >-
      github.event_name == 'push' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' && github.event.label.name == 'gpu-test')
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start.outputs.label }}
      ec2-instance-id: ${{ steps.start.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Start EC2 runner
        id: start
        uses: machulav/ec2-github-runner@v2
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          ec2-image-id: ${{ secrets.EC2_AMI_ID }}
          ec2-instance-type: ${{ inputs.ec2-instance-type || 'g4dn.4xlarge' }}
          subnet-id: ${{ secrets.EC2_SUBNET_ID }}
          security-group-id: ${{ secrets.EC2_SECURITY_GROUP_ID }}

  gpu-validation:
    name: Run GPU + CPU validation
    needs: start-runner
    runs-on: ${{ needs.start-runner.outputs.label }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential cmake git

      - name: Install CUDA toolkit
        run: |
          # Deep Learning Base AMI has the NVIDIA driver but not nvcc
          # Add CUDA to PATH if already installed
          if [ -d /usr/local/cuda/bin ]; then
            echo "/usr/local/cuda/bin" >> $GITHUB_PATH
            echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> $GITHUB_ENV
          fi

          # Check if nvcc is now available
          if command -v nvcc &>/dev/null || [ -x /usr/local/cuda/bin/nvcc ]; then
            echo "CUDA toolkit already installed"
            /usr/local/cuda/bin/nvcc --version || nvcc --version
          else
            echo "Installing CUDA toolkit from NVIDIA repository..."
            # Install NVIDIA CUDA keyring for Ubuntu 22.04
            wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
            sudo dpkg -i cuda-keyring_1.1-1_all.deb
            sudo apt-get update
            sudo apt-get install -y cuda-toolkit
            echo "/usr/local/cuda/bin" >> $GITHUB_PATH
            echo "LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> $GITHUB_ENV
          fi

      - name: Verify hardware
        run: |
          echo "=== GPU ==="
          nvidia-smi
          echo ""
          echo "=== CPU (AVX2) ==="
          if grep -q avx2 /proc/cpuinfo; then
            echo "AVX2: supported"
          else
            echo "AVX2: NOT supported"
            exit 1
          fi
          echo ""
          echo "=== CUDA ==="
          nvcc --version

      - name: Configure
        run: |
          cmake -B build -G "Unix Makefiles" \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_C_COMPILER=gcc \
            -DBUILD_TESTS=ON \
            -DBUILD_EXAMPLES=OFF \
            -DCFD_ENABLE_AVX2=ON \
            -DCFD_ENABLE_CUDA=ON \
            -DCAVITY_FULL_VALIDATION=ON

      - name: Build
        run: cmake --build build --parallel $(nproc)

      - name: Verify backends
        run: |
          cd build
          OUTPUT=$(./test_linear_solver 2>&1 || true)
          echo "$OUTPUT"

          echo ""
          echo "Checking backend availability..."

          # AVX2
          if echo "$OUTPUT" | grep -q "SIMD backend available: YES"; then
            echo "AVX2: enabled"
          else
            echo "AVX2: MISSING"
            exit 1
          fi

          # GPU symbols
          LIB_CHECK=""
          if [ -f lib/libcfd_library.a ]; then
            LIB_CHECK=lib/libcfd_library.a
          elif [ -f lib/libcfd_api.a ]; then
            LIB_CHECK=lib/libcfd_api.a
          fi
          if [ -n "$LIB_CHECK" ] && nm -C "$LIB_CHECK" 2>/dev/null | grep -q "gpu"; then
            echo "GPU: symbols found"
          else
            echo "GPU: symbols MISSING"
          fi

      - name: Run validation tests
        run: |
          cd build
          echo "=== Cross-architecture consistency ==="
          ctest --output-on-failure -L "cross-arch" -V
          echo ""
          echo "=== Physics validation (129x129 cavity) ==="
          # Distribute vCPUs evenly: nproc / 4 parallel tests
          OMP_THREADS=$(( $(nproc) / 4 ))
          OMP_THREADS=$(( OMP_THREADS > 0 ? OMP_THREADS : 1 ))
          echo "Running 4 parallel tests with OMP_NUM_THREADS=$OMP_THREADS ($(nproc) vCPUs)"
          OMP_NUM_THREADS=$OMP_THREADS ctest --output-on-failure -L "validation" -V -j 4 2>&1 | tee validation_output.txt

      - name: Verify GPU backend was tested
        run: |
          cd build
          # Check validation output for GPU skip (don't re-run the full test)
          if [ -f validation_output.txt ] && grep -q "projection_jacobi_gpu.*SKIPPED" validation_output.txt; then
            echo "WARNING: GPU backend was skipped despite GPU hardware"
            grep -i "gpu\|GPU" validation_output.txt
          else
            echo "GPU backend ran successfully (or not applicable)"
          fi

      - name: Generate summary
        if: always()
        run: |
          echo "## GPU Validation Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- **Instance**: ${{ inputs.ec2-instance-type || 'g4dn.4xlarge' }} (T4 GPU + $(nproc) vCPUs)" >> $GITHUB_STEP_SUMMARY
          echo "- **Backends**: AVX2, OpenMP, CUDA GPU" >> $GITHUB_STEP_SUMMARY
          echo "- **Grid**: 129x129 full validation" >> $GITHUB_STEP_SUMMARY
          echo "- **GPU**: $(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo 'N/A')" >> $GITHUB_STEP_SUMMARY

  stop-runner:
    name: Stop EC2 runner
    needs: [start-runner, gpu-validation]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}