test_coverage.yml

name: Test coverage
on:
  # Run manually.
  workflow_dispatch: {}
    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch
  # every day at 00:00 UTC.
  # schedule:
  #   - cron:  '0 0 * * *'

env:
  CSFY_CI: true

# Set up permissions for OIDC authentication.
permissions:
  # This is required for requesting the OIDC JWT.
  id-token: write
  # This is required for actions/checkout.
  contents: read
  # This is required for pulling the Docker image from GHCR.
  packages: read
jobs:
  run_test_coverage:
    runs-on: ubuntu-latest

    steps:
      # Configure AWS authentication for this workflow.
      # This step assumes an AWS IAM role to grant GH Action temporary
      # credentials necessary to access AWS resources.
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          role-to-assume: ${{ vars.GH_ACTION_AWS_ROLE_ARN }}
          role-session-name: ${{ vars.GH_ACTION_AWS_SESSION_NAME }}
          aws-region: ${{ vars.CSFY_AWS_DEFAULT_REGION }}

      # This is needed to pull the Docker image.
      - name: Login to GHCR
        run: docker login ghcr.io -u gpsaggese -p ${{ secrets.GH_ACTION_ACCESS_TOKEN }}

      # Make everything accessible by any user to avoid permission errors.
      - name: Cleanup
        run: sudo chmod 777 -R .

      # Check out the code from GitHub so that we can run the action inside
      # the Docker container.
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          submodules: true
          # TODO(Samarth): Do we need to propagate this to other `repos/workflow`
          # make it a default behavior? For certain tests to pass, we need entire
          # commit history of the repo including sub-modules.
          fetch-depth: 0
          token: ${{ secrets.GH_ACTION_ACCESS_TOKEN }}

      # To see the modules in `helpers_root`, PYTHONPATH needs to include
      # `helpers_root` in the same way we do in `setenv.sh`.
      - name: Update PYTHONPATH
        run: echo "PYTHONPATH=.:helpers_root" >> $GITHUB_ENV

      # Install packages that are required to run the job via GH.
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r .github/gh_requirements.txt

      # Pull the latest Docker image from the GHCR registry instead of ECR for
      # cost saving purposes to run the regressions on.
      - name: Pull image from GHCR
        run: docker pull ghcr.io/${{ github.repository }}:dev

      # This step is used to trigger the fast test coverage generation using the invoke task.
      - name: Run Fast test and generate report
        id: run_fast
        continue-on-error: true
        env:
          GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }}
          CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
          CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
          CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
          CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }}
          CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }}
          CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }}
        run: invoke run_coverage --suite fast

      - name: Upload Fast Test Coverage to Codecov
        id: upload_fast
        # Only upload if the previous fast test run step succeeded (i.r report generated).
        # failed step don’t generate a coverage report, so there's nothing to upload.
        if: steps.run_fast.outcome == 'success'
        continue-on-error: true
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml
          # Specify the Codecov flag name associated with this test suite.
          # Required to separate coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI.
          flags: fast
          name: fast-test-coverage

      - name: Run Slow test and generate report
        id: run_slow
        continue-on-error: true
        env:
          GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }}
          CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
          CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
          CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
          CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }}
          CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }}
          CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }}
        run: invoke run_coverage --suite slow

      - name: Upload Slow Test Coverage to Codecov
        id: upload_slow
        # Only upload if the previous slow test run step succeeded (i.e, if report generated).
        if: steps.run_slow.outcome == 'success'
        continue-on-error: true
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml
          flags: slow
          name: slow-test-coverage

      # - name: Run Superslow test and generate report
      #   id: run_superslow
      #   env:
      #     GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }}
      #     CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
      #     CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
      #     CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
      #     CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }}
      #     CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }}
      #     CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }}
      #   run: |
      #     # Determine the day of the week (1 = Monday, 7 = Sunday).
      #     day_of_week=$(date +%u)
      #     # Only run superslow tests on Mondays or if the workflow is manually triggered.
      #     if [ "$day_of_week" = "1" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
      #       echo "Running superslow tests..."
      #       invoke run_coverage --suite superslow
      #     else
      #       echo "Skipping superslow tests — today is not Monday and this is not a manual trigger"
      #       exit 0
      #     fi

      # - name: Upload Superslow Test Coverage to Codecov
      #   #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one.
      #   if: steps.run_superslow.outcome == 'success'
      #   uses: codecov/codecov-action@v5
      #   with:
      #     token: ${{ secrets.CODECOV_TOKEN }}
      #     files: ./coverage.xml
      #     flags: superslow
      #     name: superslow-test-coverage

      #   # Fail the job in CI if any of the fast/ slow run/ upload steps above failed.
      # - name: Fail if fast/slow test or upload failed
      #   run: |
      #     failed=""
      #     if [ "${{ steps.run_fast.outcome }}" != "success" ]; then
      #       echo "Fast test run failed"
      #       failed="true"
      #     fi
      #     if [ "${{ steps.upload_fast.outcome }}" != "success" ]; then
      #       echo "Fast test coverage upload failed"
      #       failed="true"
      #     fi
      #     if [ "${{ steps.run_slow.outcome }}" != "success" ]; then
      #       echo "Slow test run failed"
      #       failed="true"
      #     fi
      #     if [ "${{ steps.upload_slow.outcome }}" != "success" ]; then
      #       echo "Slow test coverage upload failed"
      #       failed="true"
      #     fi
      #     if [ "$failed" = "true" ]; then
      #       echo "At least one fast/slow test or upload step failed."
      #       exit 1
      #     fi