main.yml

# This is a basic workflow to help you get started with Actions

#Turing commands are commented out , we regress QV100,A100

name: CI Runs

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the mydev branch
  push:
  pull_request:
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
env:
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  PR_HEAD_REPO_FULL_NAME: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

jobs:
  check-format:
    runs-on: tgrogers-raid
    steps:
      - uses: actions/checkout@v4
      
      - name: Check code formatting
        run: |
          # Run the formatter to see if it would make any changes
          /bin/bash ./gpu-simulator/format-code.sh
          /bin/bash ./util/tracer_nvbit/tracer_tool/format-code.sh
          
          # Check if there are any uncommitted changes
          if git diff --quiet; then
            echo "✅ Code is properly formatted"
          else
            echo "❌ Code formatting check failed. The following files need formatting:"
            git diff --name-only
            echo ""
            echo "Please run the formatter locally and commit the changes:"
            echo "/bin/bash ./format-code.sh"
            exit 1
          fi
  SASS-Simulation:
    runs-on: tgrogers-raid
    needs: [check-format]
    timeout-minutes: 720
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        run: |
          rm -rf env-setup
          git clone --quiet git@github.com:purdue-aalp/env-setup.git 
          cd env-setup 
          git checkout cluster-ubuntu
      - name: Build Accel-Sim
        run: |
          source ./env-setup/12.8_env_setup.sh
          rm -rf ./gpu-simulator/gpgpu-sim
          
          # Clone gpgpu-sim with fork-aware branch selection
          echo "Cloning gpgpu-sim with fork-aware branch selection..."
          git clone --quiet git@github.com:accel-sim/gpgpu-sim_distribution.git ./gpu-simulator/gpgpu-sim
          
          # Try to checkout the same branch from the same owner's fork first
          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
            current_owner=$(echo $PR_HEAD_REPO_FULL_NAME | cut -d'/' -f1)
          else
            current_owner=$(echo ${{ github.repository }} | cut -d'/' -f1)
          fi
          current_repo=$(echo ${{ github.repository }} | cut -d'/' -f2)

          gpgpusim_repo=$(echo $current_repo | sed 's/accel-sim-framework/gpgpu-sim_distribution/')
          
          echo "Attempting to checkout branch '$BRANCH_NAME' from '$current_owner/$gpgpusim_repo'"
          
          # First, try to add the fork owner's repository as a remote and check if the branch exists
          if git -C ./gpu-simulator/gpgpu-sim/ remote add fork-owner git@github.com:$current_owner/$gpgpusim_repo.git 2>/dev/null; then
            # Check if the branch exists in the fork owner's repository
            if git -C ./gpu-simulator/gpgpu-sim/ ls-remote fork-owner | grep -q "refs/heads/$BRANCH_NAME"; then
              echo "Found branch '$BRANCH_NAME' in '$current_owner/$gpgpusim_repo' repository, checking it out"
              git -C ./gpu-simulator/gpgpu-sim/ fetch fork-owner
              git -C ./gpu-simulator/gpgpu-sim/ checkout -B $BRANCH_NAME fork-owner/$BRANCH_NAME
            else
              echo "Branch '$BRANCH_NAME' not found in '$current_owner/$gpgpusim_repo' repository, falling back to upstream dev branch"
              git -C ./gpu-simulator/gpgpu-sim/ checkout -B dev origin/dev
            fi
            # Remove the temporary remote
            git -C ./gpu-simulator/gpgpu-sim/ remote remove fork-owner
          else
            echo "Could not add '$current_owner/$gpgpusim_repo' remote, falling back to upstream dev branch"
            git -C ./gpu-simulator/gpgpu-sim/ checkout -B dev origin/dev
          fi
          
          source ./gpu-simulator/setup_environment.sh
          make clean -C gpu-simulator
          srun --time=8:00:00 -c20 make -j20 -C gpu-simulator
      - name: Run SASS
        run: |
          source ./env-setup/12.8_env_setup.sh
          source ./gpu-simulator/setup_environment.sh
          ./util/job_launching/run_simulations.py -B GPU_Microbenchmark -C QV100-SASS -T /scratch/tgrogers-disk01/a/common/for-sharing/accel-sim/QV100//hw_run/traces/device-0/12.8/ -N sass-short-${{ github.run_number }}"_"${{ github.run_attempt}}
          ./util/job_launching/run_simulations.py -B rodinia_2.0-ft  -C QV100-SASS -T /scratch/tgrogers-disk01/a/common/for-sharing/accel-sim/QV100/hw_run/traces/device-0/12.8/ -N sass-short-${{ github.run_number }}"_"${{ github.run_attempt}}

          ./util/job_launching/run_simulations.py -B GPU_Microbenchmark -C A100-SASS -T /scratch/tgrogers-disk01/a/common/for-sharing/accel-sim/A100/hw_run/traces/device-0/12.8/  -N sass-short-${{ github.run_number }}"_"${{ github.run_attempt}}
          ./util/job_launching/run_simulations.py -B rodinia_2.0-ft -C A100-SASS -T /scratch/tgrogers-disk01/a/common/for-sharing/accel-sim/A100/hw_run/traces/device-0/12.8/  -N sass-short-${{ github.run_number }}"_"${{ github.run_attempt}}

          ./util/job_launching/monitor_func_test.py -v --sleep_time 300 -s stats-per-app-sass.csv -N sass-short-${{ github.run_number }}"_"${{ github.run_attempt}}
      - name: Archive Stats
        run: |
          source ./env-setup/12.8_env_setup.sh
          rm -rf ./statistics-archive
          git clone --quiet git@github.com:accel-sim/statistics-archive.git
          BRANCH_NAME=$PR_HEAD_REPO_FULL_NAME/$BRANCH_NAME
          # either create a new branch or check it out if it already exists
          git -C ./statistics-archive checkout $BRANCH_NAME 2>/dev/null || git -C ./statistics-archive checkout -b $BRANCH_NAME
          ./util/job_launching/get_stats.py -k -K -R -B GPU_Microbenchmark -C QV100-SASS -A | tee v100-ubench-sass-local.csv
          ./util/job_launching/get_stats.py -k -K -R -B GPU_Microbenchmark -C A100-SASS -A | tee ampere-a100-ubench-sass-local.csv

          mkdir -p statistics-archive/ubench/
          # First we merge and archive this run to the main csv that contains all previous runs
          ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/v100-ubench-sass.csv,v100-ubench-sass-local.csv \
          | tee v100-ubench-sass.csv && mv v100-ubench-sass.csv ./statistics-archive/ubench/
          # ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/turing-ubench-sass.csv,turing-ubench-sass-local.csv \
          # | tee turing-ubench-sass.csv && mv turing-ubench-sass.csv ./statistics-archive/ubench/
          ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/ampere-a100-ubench-sass.csv,ampere-a100-ubench-sass-local.csv \
          | tee ampere-a100-ubench-sass.csv && mv ampere-a100-ubench-sass.csv ./statistics-archive/ubench/
          # Next we merge the latest run with the current run (used for correlation plots) then archive the current run as the new latest for the next time this action occurs
          ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/v100-ubench-sass-latest.csv,v100-ubench-sass-local.csv \
          | tee v100-ubench-sass-latest2.csv && mv v100-ubench-sass-local.csv ./statistics-archive/ubench/v100-ubench-sass-latest.csv
          # ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/turing-ubench-sass-latest.csv,turing-ubench-sass-local.csv \
          # | tee turing-ubench-sass-latest2.csv && mv turing-ubench-sass-local.csv ./statistics-archive/ubench/turing-ubench-sass-latest.csv
          ./util/plotting/merge-stats.py -R -c ./statistics-archive/ubench/ampere-a100-ubench-sass-latest.csv,ampere-a100-ubench-sass-local.csv \
          | tee ampere-a100-ubench-sass-latest2.csv && mv ampere-a100-ubench-sass-local.csv ./statistics-archive/ubench/ampere-a100-ubench-sass-latest.csv
          git -C ./statistics-archive add --all
          git -C ./statistics-archive commit \
          -m "CI automated checkin $BRANCH_NAME Build:${{ github.run_number }}"_"${{ github.run_attempt}}" || echo "No Changes."
          git -C ./statistics-archive push -u origin $BRANCH_NAME
      - name: Correlate Ubench
        run: |
          source ./env-setup/12.8_env_setup.sh
          ./util/hw_stats/get_hw_data.sh > /dev/null 2>&1
          rm -rf ./util/plotting/correl-html/
          ./util/plotting/plot-correlation.py -c ./v100-ubench-sass-latest2.csv -H /scratch/tgrogers-disk01/a/common/for-sharing/accel-sim/QV100/hw_run/device-0/12.8/ | tee v100-ubench-correl.txt
          mv ./util/plotting/correl-html/combined_per_kernel.html ./statistics-archive/ubench/v100-combined_per_kernel.html
          mv ./util/plotting/correl-html/combined_per_app.html ./statistics-archive/ubench/v100-combined_per_app.html

          ./util/plotting/plot-correlation.py -c ./ampere-a100-ubench-sass-latest2.csv -H /scratch/tgrogers-disk01/a/common/for-sharing/accel-sim/A100/hw_run/device-0/12.8/ | tee ampere-a100-ubench-correl.txt
          mv ./util/plotting/correl-html/combined_per_kernel.html ./statistics-archive/ubench/ampere-a100-combined_per_kernel.html
          mv ./util/plotting/correl-html/combined_per_app.html ./statistics-archive/ubench/ampere-a100-combined_per_app.html

          ssh ghci@tgrogers-pc01 mkdir -p /home/ghci/accel-sim/correl/git_$BRANCH_NAME"_"${{ github.run_number }}"_"${{ github.run_attempt}}/
          rsync --delete -r ./util/plotting/correl-html/ ghci@tgrogers-pc01:/home/ghci/accel-sim/correl/git_$BRANCH_NAME"_"${{ github.run_number }}"_"${{ github.run_attempt}}/
          git -C ./statistics-archive reset --soft HEAD~1
          git -C ./statistics-archive add --all
          git -C ./statistics-archive commit \
          -m "CI automated checkin $PR_HEAD_REPO_FULL_NAME/$BRANCH_NAME Build:${{ github.run_number }}"_"${{ github.run_attempt}}" || echo "No Changes."
          git -C ./statistics-archive push -f -u origin $PR_HEAD_REPO_FULL_NAME/$BRANCH_NAME
          if [[ ${{ github.event_name }} == 'push' ]]; then

            rm -rf /scratch/tgrogers-disk01/a/tgrghci/ci/lastSuccess/${{ github.repository }}/$BRANCH_NAME
            mkdir -p /scratch/tgrogers-disk01/a/tgrghci/ci/lastSuccess/${{ github.repository }}/$BRANCH_NAME
            mv $PWD/* /scratch/tgrogers-disk01/a/tgrghci/ci/lastSuccess/${{ github.repository }}/$BRANCH_NAME/
            chmod -R 777 /scratch/tgrogers-disk01/a/tgrghci/ci/lastSuccess/${{ github.repository }}/$BRANCH_NAME/
          fi
  PTX-Simulation:
    needs: [check-format]
    runs-on: tgrogers-raid
    timeout-minutes: 720
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        run: |
          rm -rf env-setup
          git clone --quiet git@github.com:purdue-aalp/env-setup.git 
          cd env-setup 
          git checkout cluster-ubuntu
      - name: Build Accel-Sim
        run: |
          source ./env-setup/12.8_env_setup.sh
          rm -rf ./gpu-simulator/gpgpu-sim
          
          # Clone gpgpu-sim with fork-aware branch selection
          echo "Cloning gpgpu-sim with fork-aware branch selection..."
          git clone --quiet git@github.com:accel-sim/gpgpu-sim_distribution.git ./gpu-simulator/gpgpu-sim
          
          # Try to checkout the same branch from the same owner's fork first
          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
            current_owner=$(echo $PR_HEAD_REPO_FULL_NAME | cut -d'/' -f1)
          else
            current_owner=$(echo ${{ github.repository }} | cut -d'/' -f1)
          fi
          current_branch=$BRANCH_NAME
          current_repo=$(echo $GITHUB_REPOSITORY | cut -d'/' -f2)
          
          gpgpusim_repo=$(echo $current_repo | sed 's/accel-sim-framework/gpgpu-sim_distribution/')

          echo "Attempting to checkout branch '$BRANCH_NAME' from '$current_owner/$gpgpusim_repo'"
          
          # First, try to add the fork owner's repository as a remote and check if the branch exists
          if git -C ./gpu-simulator/gpgpu-sim/ remote add fork-owner git@github.com:$current_owner/$gpgpusim_repo.git 2>/dev/null; then
            # Check if the branch exists in the fork owner's repository
            if git -C ./gpu-simulator/gpgpu-sim/ ls-remote fork-owner | grep -q "refs/heads/$BRANCH_NAME"; then
              echo "Found branch '$BRANCH_NAME' in '$current_owner/$gpgpusim_repo' repository, checking it out"
              git -C ./gpu-simulator/gpgpu-sim/ fetch fork-owner
              git -C ./gpu-simulator/gpgpu-sim/ checkout -B $BRANCH_NAME fork-owner/$BRANCH_NAME
            else
              echo "Branch '$BRANCH_NAME' not found in '$current_owner/$gpgpusim_repo' repository, falling back to accel-sim dev branch"
              git -C ./gpu-simulator/gpgpu-sim/ checkout -B dev origin/dev
            fi
            # Remove the temporary remote
            git -C ./gpu-simulator/gpgpu-sim/ remote remove fork-owner
          else
            echo "Could not add '$current_owner/$gpgpusim_repo' remote, falling back to upstream dev branch"
            git -C ./gpu-simulator/gpgpu-sim/ checkout -B dev origin/dev
          fi
          
          source ./gpu-simulator/setup_environment.sh
          make clean -C gpu-simulator
          srun --time=8:00:00 -c20 make -j20 -C gpu-simulator
      - name: Run PTX
        run: |
          source ./env-setup/12.8_env_setup.sh
          source ./gpu-simulator/setup_environment.sh

          rm -rf ./gpu-app-collection
          git clone --quiet --recurse-submodules -b dev git@github.com:accel-sim/gpu-app-collection.git
          source ./gpu-app-collection/src/setup_environment > /dev/null 2>&1
          srun --time=8:00:00 -c20 make rodinia_2.0-ft GPU_Microbenchmark -j20 -C ./gpu-app-collection/src
          ./gpu-app-collection/get_regression_data.sh

          #./util/job_launching/run_simulations.py -B rodinia_2.0-ft,GPU_Microbenchmark -C QV100-PTX,A100-PTX -N short-ptx-${{ github.run_number }}_${{ github.run_attempt}}
          ./util/job_launching/run_simulations.py -B rodinia_2.0-ft,GPU_Microbenchmark -C QV100-PTX,A100-PTX -N short-ptx-${{ github.run_number }}_${{ github.run_attempt}}

          ./util/job_launching/monitor_func_test.py -v --sleep_time 300 -s stats-per-app-ptx.csv -N short-ptx-${{ github.run_number }}_${{ github.run_attempt}}
  Tracer-Tool:
    needs: [check-format]
    runs-on: tgrogers-raid
    timeout-minutes: 720
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        run: |
          rm -rf env-setup
          git clone --quiet git@github.com:purdue-aalp/env-setup.git 
          cd env-setup 
          git checkout cluster-ubuntu
      - name: Build Accel-Sim
        run: |
          source ./env-setup/12.8_env_setup.sh
          rm -rf ./gpu-simulator/gpgpu-sim

          # Clone gpgpu-sim with fork-aware branch selection
          echo "Cloning gpgpu-sim with fork-aware branch selection..."
          git clone --quiet git@github.com:accel-sim/gpgpu-sim_distribution.git ./gpu-simulator/gpgpu-sim

          # Try to checkout the same branch from the same owner's fork first
          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
            current_owner=$(echo $PR_HEAD_REPO_FULL_NAME | cut -d'/' -f1)
          else
            current_owner=$(echo ${{ github.repository }} | cut -d'/' -f1)
          fi
          current_repo=$(echo ${{ github.repository }} | cut -d'/' -f2)

          gpgpusim_repo=$(echo $current_repo | sed 's/accel-sim-framework/gpgpu-sim_distribution/')

          echo "Attempting to checkout branch '$BRANCH_NAME' from '$current_owner/$gpgpusim_repo'"

          # First, try to add the fork owner's repository as a remote and check if the branch exists
          if git -C ./gpu-simulator/gpgpu-sim/ remote add fork-owner git@github.com:$current_owner/$gpgpusim_repo.git 2>/dev/null; then
            # Check if the branch exists in the fork owner's repository
            if git -C ./gpu-simulator/gpgpu-sim/ ls-remote fork-owner | grep -q "refs/heads/$BRANCH_NAME"; then
              echo "Found branch '$BRANCH_NAME' in '$current_owner/$gpgpusim_repo' repository, checking it out"
              git -C ./gpu-simulator/gpgpu-sim/ fetch fork-owner
              git -C ./gpu-simulator/gpgpu-sim/ checkout -B $BRANCH_NAME fork-owner/$BRANCH_NAME
            else
              echo "Branch '$BRANCH_NAME' not found in '$current_owner/$gpgpusim_repo' repository, falling back to upstream dev branch"
              git -C ./gpu-simulator/gpgpu-sim/ checkout -B dev origin/dev
            fi
            # Remove the temporary remote
            git -C ./gpu-simulator/gpgpu-sim/ remote remove fork-owner
          else
            echo "Could not add '$current_owner/$gpgpusim_repo' remote, falling back to upstream dev branch"
            git -C ./gpu-simulator/gpgpu-sim/ checkout -B dev origin/dev
          fi

          source ./gpu-simulator/setup_environment.sh
          make clean -C gpu-simulator
          make -j20 -C gpu-simulator
      - name: Build Tracer
        run: |
          source ./env-setup/12.8_env_setup.sh
          ./util/tracer_nvbit/install_nvbit.sh
          make clean -C ./util/tracer_nvbit/
          make -C ./util/tracer_nvbit/
      - name: rodinia_2.0-ft-build
        run: |
          source ./env-setup/12.8_env_setup.sh
          rm -rf ./gpu-app-collection/
          git clone --quiet git@github.com:accel-sim/gpu-app-collection.git
          source ./gpu-app-collection/src/setup_environment > /dev/null 2>&1
          ln -s /home/tgrogers-raid/a/common/data_dirs ./gpu-app-collection/
          make -C ./gpu-app-collection/src rodinia_2.0-ft
      - name: generate-rodinia_2.0-ft-traces
        run: |
          source ./env-setup/12.8_env_setup.sh
          source ./gpu-app-collection/src/setup_environment
          rm -rf ./hw_run/
          srun --job-name=gpu-lock --dependency=singleton --partition=tgrogers-dgx -- ./util/tracer_nvbit/run_hw_trace.py -B rodinia_2.0-ft -D 7
      - name: generate-rodinia_2.0-ft-hw_stats
        run: |
          source ./env-setup/12.8_env_setup.sh
          source ./gpu-app-collection/src/setup_environment
          srun --job-name=gpu-lock --dependency=singleton --partition=tgrogers-dgx -- ./util/hw_stats/run_hw.py -B rodinia_2.0-ft -D 7
      - name: test-new-traces
        run: |
          source ./env-setup/12.8_env_setup.sh
          source ./gpu-simulator/setup_environment.sh
          ./util/job_launching/run_simulations.py -B rodinia_2.0-ft -C QV100-SASS -T ./hw_run/traces/device-7/ -N rodinia_2.0-ft-$$
          ./util/job_launching/monitor_func_test.py --sleep_time 300 -v -s rodinia-stats-per-app.csv -N rodinia_2.0-ft-$$
  SASS-Simulation-Docker:
    needs: [check-format]
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/accel-sim/accel-sim-framework:ubuntu-24.04-cuda-12.8-minimal
      # env:
        
    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v4
      - name: Run Simulation
        run: /bin/bash $GITHUB_WORKSPACE/short-tests.sh
  CI-Success:
    env:
      ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
    runs-on: tgrogers-raid
    needs: [check-format, SASS-Simulation, PTX-Simulation, Tracer-Tool, SASS-Simulation-Docker]
    steps:
      - uses: actions/checkout@v4
      - run: |
          echo "✅ All CI tests completed successfully!"
          echo "check-format: ✅"
          echo "SASS-Simulation: ✅"
          echo "PTX-Simulation: ✅"
          echo "Tracer-Tool: ✅"
          echo "SASS-Simulation-Docker: ✅"
          # Setup envs
          git clone --quiet --branch cluster-ubuntu git@github.com:purdue-aalp/env-setup.git 
          source ./env-setup/common/common_inc.sh
          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
            # stats are only archived on pushes. So the repo in the report url needs to be the head repo of the PR
            # The stats htmls are generated by CI triggered by pushes to the head repo.
            # Not so clean. But works for now. The htmls are not that important anyway. 
            export REPORT_URL="https://rawcdn.githack.com/accel-sim/statistics-archive/$PR_HEAD_REPO_FULL_NAME/$BRANCH_NAME/ubench/"
            python3 .github/scripts/send_ci_email.py -t success 
          fi

  failures:
    if: failure()
    env:
      ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
      REPORT_URL: ""
    runs-on: tgrogers-raid
    needs: [check-format, SASS-Simulation, PTX-Simulation, Tracer-Tool, SASS-Simulation-Docker]
    steps:
      - uses: actions/checkout@v4
      - name: Notify Failure
        run: |
          # Setup envs
          git clone --quiet --branch cluster-ubuntu git@github.com:purdue-aalp/env-setup.git 
          source ./env-setup/common/common_inc.sh
          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
            python3 .github/scripts/send_ci_email.py -t failure
          elif [[ ${{ github.event_name }} == 'push' && ${{ github.ref_name }} == 'dev' ]]; then
             # if the push is to the dev branch, send an urgent failure email
             python3 .github/scripts/send_ci_email.py -t failure -u
          fi