diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9da6e401a3b..f23e66d37c1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,496 +1,266 @@
-name: test
-
-on:
- push:
- branches:
- - main
- - "[0-9]+.[0-9]+.x"
- - "test-me-*"
- tags:
- - "[0-9]+.[0-9]+.[0-9]+"
- - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
-
- pull_request:
- branches:
- - main
- - "[0-9]+.[0-9]+.x"
- types: [opened, synchronize, reopened, ready_for_review]
-
-env:
- PYTEST_ADDOPTS: "--color=yes"
- # SETUPTOOLS_SCM_PRETEND_VERSION: "7.3.1.dev0"
- # SETUPTOOLS_SCM_NO_LOCAL_VERSION: "1"
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- actions: read
-
-jobs:
- run-tests:
- runs-on: ${{ matrix.os }}
- permissions:
- contents: read
-
- strategy:
- fail-fast: false
- matrix:
- name: [
- "windows-py39-unittestextras",
- "windows-py39-pluggy",
- "windows-py39-xdist",
- "windows-py310",
- "windows-py311",
- "windows-py312",
- "windows-py313"
- ]
-
- include:
- - name: "windows-py39-unittestextras"
- python: "3.9"
- os: windows-latest
- tox_env: "py39-unittestextras"
-
- - name: "windows-py39-pluggy"
- python: "3.9"
- os: windows-latest
- tox_env: "py39-pluggymain-pylib-xdist"
-
- - name: "windows-py39-xdist"
- python: "3.9"
- os: windows-latest
- tox_env: "py39-xdist"
-
- - name: "windows-py310"
- python: "3.10"
- os: windows-latest
- tox_env: "py310-xdist"
-
- - name: "windows-py311"
- python: "3.11"
- os: windows-latest
- tox_env: "py311"
-
- - name: "windows-py312"
- python: "3.12"
- os: windows-latest
- tox_env: "py312"
-
- - name: "windows-py313"
- python: "3.13"
- os: windows-latest
- tox_env: "py313"
-
- steps:
- - name: Check out code
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- python-version: ${{ matrix.python }}
-
- - name: Install dependencies
- shell: bash
- run: |
- python -m pip install --upgrade pip
- pip install tox pytest-json-report jq
-
- - name: Get PR ID
- shell: bash
- if: github.event_name == 'pull_request'
- run: echo "PR_ID=${{ github.event.number }}" >> $GITHUB_ENV
-
- - name: Set Default Folder for Non-PR Runs
- shell: bash
- if: github.event_name != 'pull_request'
- run: echo "PR_ID=main" >> $GITHUB_ENV
-
- - name: Check If Previous Artifacts Exist
- id: check_artifacts
- shell: bash
- run: |
- echo "Checking if previous test results exist for PR-${PR_ID}..."
- ARTIFACTS_RESPONSE=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
- -H "Accept: application/vnd.github.v3+json" \
- "https://api.github.com/repos/${{ github.repository }}/actions/artifacts")
-
- ARTIFACT_COUNT=$(echo "$ARTIFACTS_RESPONSE" | jq -r --arg PR "pr-${PR_ID}-test-results" \
- '[.artifacts[] | select(.name==$PR)] | length')
-
- if [[ "$ARTIFACT_COUNT" -gt 0 ]]; then
- echo "PREV_ARTIFACT_EXISTS=true" >> $GITHUB_ENV
- else
- echo "PREV_ARTIFACT_EXISTS=false" >> $GITHUB_ENV
- fi
-
- - name: Retrieve Previous Artifacts (If Exists)
- if: env.PREV_ARTIFACT_EXISTS == 'true'
- shell: bash
- run: |
- echo "Fetching previous test results for PR ${PR_ID}..."
-
- ARTIFACT_URL=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
- -H "Accept: application/vnd.github.v3+json" \
- "https://api.github.com/repos/${{ github.repository }}/actions/artifacts" | \
- jq -r --arg PR "pr-${PR_ID}-test-results" \
- '[.artifacts[] | select(.name==$PR)] | sort_by(.created_at) | reverse | .[0].archive_download_url')
-
- if [[ -n "$ARTIFACT_URL" && "$ARTIFACT_URL" != "null" ]]; then
- echo "Latest artifact found. Downloading..."
- mkdir -p artifacts/pr-${PR_ID}
- curl -L -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
- -o artifacts/pr-${PR_ID}/test-results.zip "$ARTIFACT_URL"
- unzip -o artifacts/pr-${PR_ID}/test-results.zip -d artifacts/pr-${PR_ID}
-
- echo "======================================="
- echo "Previous Test Results for PR-${PR_ID}:"
- cat artifacts/pr-${PR_ID}/test_results.json || echo "No previous test results found."
- echo "======================================="
- else
- echo "No previous test results found for PR-${PR_ID}. Running fresh tests."
- fi
-
- - name: Extract Failed and Passed Tests from Previous Run
- shell: bash
- run: |
- mkdir -p artifacts/pr-${PR_ID}
- PREV_RESULTS="artifacts/pr-${PR_ID}/test_results.json"
- FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt"
- ALL_TESTS_FILE="artifacts/pr-${PR_ID}/all_tests.txt"
- REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt"
-
- # Use tox to collect all tests
- tox -e ${{ matrix.tox_env }} -- --collect-only --quiet | grep -v "SKIP" | grep "::" > $ALL_TESTS_FILE || true
- #tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep -E "^(.*?)::" | sed -E 's/\s+.*$//' > $ALL_TESTS_FILE || true
-
- if [[ -f "$PREV_RESULTS" ]]; then
- echo "Extracting failed test cases from previous run..."
- cat $PREV_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_TESTS_FILE
- else
- echo "No previous test results found. Skipping extraction."
- touch $FAILED_TESTS_FILE
- fi
-
- if [[ -s "$FAILED_TESTS_FILE" ]]; then
- echo "Failed tests from the previous run:"
- cat $FAILED_TESTS_FILE
- else
- echo "No previously failed tests found."
- fi
-
- - name: Pre-Check for Previously Failed Tests
- shell: bash
- run: |
- FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt"
- SKIPPED_TESTS_FILE="artifacts/pr-${PR_ID}/skipped_tests.txt"
-
- # Only run this check if we have previously failed tests
- if [[ -s "$FAILED_TESTS_FILE" ]]; then
- echo "Checking for skipped tests among previously failed tests..."
- tox -e ${{ matrix.tox_env }} -- --collect-only -v $(cat $FAILED_TESTS_FILE) | grep "SKIP" | grep "::" | sed 's/.*SKIP //g' > $SKIPPED_TESTS_FILE
-
- # Remove skipped tests from the failed tests list
- if [[ -s "$SKIPPED_TESTS_FILE" ]]; then
- echo "Removing skipped tests from the rerun list:"
- cat $SKIPPED_TESTS_FILE
- grep -v -F -f $SKIPPED_TESTS_FILE $FAILED_TESTS_FILE > "artifacts/pr-${PR_ID}/filtered_failed_tests.txt"
- mv "artifacts/pr-${PR_ID}/filtered_failed_tests.txt" $FAILED_TESTS_FILE
- else
- echo "No skipped tests found among previously failed tests."
- fi
- fi
-
- - name: Generate Failed Test Commands
- shell: bash
- run: |
- FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt"
-
- if [[ -s "$FAILED_TESTS_FILE" ]]; then
- python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 --tox-env ${{ matrix.tox_env }}
- fi
-
- - name: Run Previously Failed Tests First
- shell: bash
- run: |
- FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt"
-
- if [[ -s "$FAILED_TESTS_FILE" ]]; then
- echo "Rerunning previously failed tests using tox env ${{ matrix.tox_env }}..."
-
- if [[ -f "artifacts/pr-${PR_ID}/run_failed_tests.sh" ]]; then
- chmod +x artifacts/pr-${PR_ID}/run_failed_tests.sh
- bash artifacts/pr-${PR_ID}/run_failed_tests.sh
- else
- echo "No failed test script generated."
- fi
- else
- echo "No previously failed tests found."
- fi
-
-
- - name: Check If Any Tests Failed Again
- shell: bash
- run: |
- TEMP_RESULTS="artifacts/pr-${PR_ID}/temp_test_results.json"
- FAILED_AGAIN_FILE="artifacts/pr-${PR_ID}/failed_again.txt"
-
- if [[ -f "$TEMP_RESULTS" ]]; then
- echo "Analyzing test results..."
- # Extract failed tests (excluding skipped)
- cat $TEMP_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_AGAIN_FILE
- # Extract skipped tests for reporting
- cat $TEMP_RESULTS | jq -r '.tests | map(select(.outcome == "skipped")) | .[].nodeid' > "artifacts/pr-${PR_ID}/skipped_tests_report.txt"
-
- # Report on skipped tests
- if [[ -s "artifacts/pr-${PR_ID}/skipped_tests_report.txt" ]]; then
- echo "The following tests were skipped during execution:"
- cat "artifacts/pr-${PR_ID}/skipped_tests_report.txt"
- fi
- fi
-
- if [[ -s "$FAILED_AGAIN_FILE" ]]; then
- echo "Some tests failed again. Stopping execution."
- exit 1
- fi
-
- - name: Identify Remaining Untested Test Cases
- shell: bash
- run: |
- FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt"
- ALL_TESTS_FILE="artifacts/pr-${PR_ID}/all_tests.txt"
- REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt"
-
- echo "Finding remaining tests to run..."
- grep -v -F -f $FAILED_TESTS_FILE $ALL_TESTS_FILE > $REMAINING_TESTS_FILE || true
-
- if [[ -s "$REMAINING_TESTS_FILE" ]]; then
- echo "Remaining tests to run:"
- cat $REMAINING_TESTS_FILE
- else
- echo "No remaining tests to run."
- fi
-
- - name: Set Workflow ID
- shell: bash
- run: echo "WORKFLOW_ID=${{ matrix.name }}" >> $GITHUB_ENV
-
- - name: Generate Test Commands
- shell: bash
- run: |
- python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --workflow-id ${WORKFLOW_ID} --generate-script --batch-size 20 --tox-env ${{ matrix.tox_env }}
-
- - name: Display Retrieved Test Results
- shell: bash
- run: |
- RUN_TESTS_FILE="artifacts/pr-${PR_ID}/run_tests.sh"
- if [[ -f "$RUN_TESTS_FILE" ]]; then
- echo "Content of run_tests.sh:"
- cat "$RUN_TESTS_FILE"
- else
- echo "run_tests.sh file does not exist."
- fi
-
- - name: Run Remaining Test Cases
- shell: bash
- run: |
- REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt"
-
- if [[ -s "$REMAINING_TESTS_FILE" ]]; then
- echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..."
-
- if [[ -f "artifacts/pr-${PR_ID}/${WORKFLOW_ID}/run_tests.sh" ]]; then
- chmod +x artifacts/pr-${PR_ID}/${WORKFLOW_ID}/run_tests.sh
- bash artifacts/pr-${PR_ID}/${WORKFLOW_ID}/run_tests.sh
-
- # Combine results after running tests
- python scripts/generate_pytest_commands.py --combine-results --output-dir=artifacts --pr-id=${PR_ID} --workflow-id=${WORKFLOW_ID}
- else
- echo "No test script generated."
- fi
- else
- echo "No remaining tests to run."
- fi
-
- - name: Upload New Test Results
- uses: actions/upload-artifact@v4
- with:
- name: pr-${{ env.PR_ID }}-${{ env.WORKFLOW_ID }}-test-results
- path: |
- artifacts/pr-${{ env.PR_ID }}/${{ env.WORKFLOW_ID }}/test_results.json
- artifacts/pr-${{ env.PR_ID }}/${{ env.WORKFLOW_ID }}/*.sh
-
- retrieve-results:
- needs: run-tests
- runs-on: ubuntu-latest
- steps:
- - name: Get PR ID
- if: github.event_name == 'pull_request'
- run: echo "PR_ID=${{ github.event.number }}" >> $GITHUB_ENV
-
- - name: Set Default Folder for Non-PR Runs
- if: github.event_name != 'pull_request'
- run: echo "PR_ID=main" >> $GITHUB_ENV
-
- - name: Check out code
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- python-version: "3.9"
-
- - name: Install dependencies
- shell: bash
- run: |
- python -m pip install --upgrade pip
- pip install jq
-
- - name: Create directory for results
- run: mkdir -p retrieved-results
-
- - name: Download all workflow test results
- uses: actions/download-artifact@v4
- with:
- pattern: pr-${{ env.PR_ID }}-*-test-results
- path: retrieved-results
- merge-multiple: false
-
- - name: Debug directory structure
- shell: bash
- run: |
- echo "Debugging directory structure..."
- find retrieved-results -type f | sort
-
- - name: Extract zip files if needed
- shell: bash
- run: |
- echo "Extracting downloaded artifacts..."
- for zip_file in $(find retrieved-results -name "*.zip"); do
- if [ -f "$zip_file" ]; then
- workflow_dir=$(dirname "$zip_file")
- echo "Extracting $zip_file to $workflow_dir"
- unzip -o "$zip_file" -d "$workflow_dir"
- fi
- done
-
- - name: Find test result files
- shell: bash
- run: |
- echo "======================================="
- echo "Downloaded artifacts for PR ${PR_ID}:"
- find retrieved-results -type f -name "test_results*.json" | sort
- echo "======================================="
-
- - name: Combine test results
- shell: bash
- run: |
- echo "Combining test results from all workflows..."
-
- # Initialize combined results file
- cat > retrieved-results/combined_results.json << EOF
- {
- "created": "$(date -Iseconds)",
- "duration": 0,
- "exitcode": 0,
- "summary": {
- "passed": 0,
- "failed": 0,
- "skipped": 0,
- "xfailed": 0,
- "xpassed": 0,
- "error": 0,
- "total": 0
- },
- "tests": [],
- "collectors": [],
- "warnings": []
- }
- EOF
-
- # Find all test_results.json files
- for result_file in $(find retrieved-results -type f -name "test_results*.json"); do
- echo "Processing $result_file"
-
- # Check if file is valid JSON
- if ! jq empty "$result_file" 2>/dev/null; then
- echo "Warning: $result_file is not valid JSON, skipping"
- continue
- fi
-
- # Update summary counts
- for metric in passed failed skipped xfailed xpassed error total; do
- count=$(jq -r ".summary.$metric // 0" "$result_file")
- current=$(jq -r ".summary.$metric" retrieved-results/combined_results.json)
- new_count=$((current + count))
- jq --arg metric "$metric" --argjson count "$new_count" '.summary[$metric] = $count' retrieved-results/combined_results.json > temp.json && mv temp.json retrieved-results/combined_results.json
- done
-
- # Add tests
- jq -s '.[0].tests = (.[0].tests + (.[1].tests // [])); .[0]' retrieved-results/combined_results.json "$result_file" > temp.json && mv temp.json retrieved-results/combined_results.json
-
- # Add duration
- duration=$(jq -r ".duration // 0" "$result_file")
- current_duration=$(jq -r ".duration" retrieved-results/combined_results.json)
- new_duration=$(echo "$current_duration + $duration" | bc)
- jq --argjson duration "$new_duration" '.duration = $duration' retrieved-results/combined_results.json > temp.json && mv temp.json retrieved-results/combined_results.json
-
- # Update exitcode (non-zero takes precedence)
- exitcode=$(jq -r ".exitcode // 0" "$result_file")
- current_exitcode=$(jq -r ".exitcode" retrieved-results/combined_results.json)
- if [ "$exitcode" -ne 0 ] && [ "$current_exitcode" -eq 0 ]; then
- jq --argjson exitcode "$exitcode" '.exitcode = $exitcode' retrieved-results/combined_results.json > temp.json && mv temp.json retrieved-results/combined_results.json
- fi
- done
-
- # Create a copy as test_results.json for backward compatibility
- cp retrieved-results/combined_results.json retrieved-results/test_results.json
-
- - name: Display Combined Test Results
- shell: bash
- run: |
- echo "======================================="
- echo "Combined Test Results from PR ${PR_ID}:"
- echo "Summary:"
- jq '.summary' retrieved-results/combined_results.json
- echo "Failed Tests:"
- jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' retrieved-results/combined_results.json || echo "No failed tests found."
- echo "======================================="
-
- - name: Upload Combined Results
- uses: actions/upload-artifact@v4
- with:
- name: pr-${{ env.PR_ID }}-combined-test-results
- path: retrieved-results/combined_results.json
-
- # # retrieve-results:
- # needs: run-tests
- # runs-on: ubuntu-latest
- # steps:
- # - name: Get PR ID
- # if: github.event_name == 'pull_request'
- # run: echo "PR_ID=${{ github.event.number }}" >> $GITHUB_ENV
-
- # - name: Set Default Folder for Non-PR Runs
- # if: github.event_name != 'pull_request'
- # run: echo "PR_ID=main" >> $GITHUB_ENV
-
- # - name: Download Test Results
- # uses: actions/download-artifact@v4
- # with:
- # name: pr-${{ env.PR_ID }}-test-results
- # path: retrieved-results
-
- # - name: Display Retrieved Test Results
- # shell: bash
- # run: |
- # echo "======================================="
- # echo "Retrieved Test Results from PR ${PR_ID}:"
- # cat retrieved-results/test_results.json
- # echo "======================================="
+# name: test
+
+# on:
+# push:
+# branches:
+# - main
+# - "[0-9]+.[0-9]+.x"
+# tags:
+# - "[0-9]+.[0-9]+.[0-9]+"
+# - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
+
+# pull_request:
+# branches:
+# - main
+# - "[0-9]+.[0-9]+.x"
+# types:
+# - opened # default
+# - synchronize # default
+# - reopened # default
+# - ready_for_review # used in PRs created from the release workflow
+
+# env:
+# PYTEST_ADDOPTS: "--color=yes"
+
+# # Cancel running jobs for the same workflow and branch.
+# concurrency:
+# group: ${{ github.workflow }}-${{ github.ref }}
+# cancel-in-progress: true
+
+# # Set permissions at the job level.
+# permissions: {}
+
+# jobs:
+# package:
+# runs-on: ubuntu-latest
+# steps:
+# - uses: actions/checkout@v4
+# with:
+# fetch-depth: 0
+# persist-credentials: false
+# - name: Build and Check Package
+# uses: hynek/build-and-inspect-python-package@v2.10.0
+
+# build:
+# needs: [package]
+
+# runs-on: ${{ matrix.os }}
+# timeout-minutes: 45
+# permissions:
+# contents: read
+
+# strategy:
+# fail-fast: false
+# matrix:
+# name: [
+# "windows-py38",
+# "windows-py38-pluggy",
+# "windows-py39",
+# "windows-py310",
+# "windows-py311",
+# "windows-py312",
+# "windows-py313",
+
+# "ubuntu-py38",
+# "ubuntu-py38-pluggy",
+# "ubuntu-py38-freeze",
+# "ubuntu-py39",
+# "ubuntu-py310",
+# "ubuntu-py311",
+# "ubuntu-py312",
+# "ubuntu-py313",
+# "ubuntu-pypy3",
+
+# "macos-py38",
+# "macos-py39",
+# "macos-py310",
+# "macos-py312",
+# "macos-py313",
+
+# "doctesting",
+# "plugins",
+# ]
+
+# include:
+# - name: "windows-py38"
+# python: "3.8"
+# os: windows-latest
+# tox_env: "py38-unittestextras"
+# use_coverage: true
+# - name: "windows-py38-pluggy"
+# python: "3.8"
+# os: windows-latest
+# tox_env: "py38-pluggymain-pylib-xdist"
+# - name: "windows-py39"
+# python: "3.9"
+# os: windows-latest
+# tox_env: "py39-xdist"
+# - name: "windows-py310"
+# python: "3.10"
+# os: windows-latest
+# tox_env: "py310-xdist"
+# - name: "windows-py311"
+# python: "3.11"
+# os: windows-latest
+# tox_env: "py311"
+# - name: "windows-py312"
+# python: "3.12"
+# os: windows-latest
+# tox_env: "py312"
+# - name: "windows-py313"
+# python: "3.13-dev"
+# os: windows-latest
+# tox_env: "py313"
+
+# - name: "ubuntu-py38"
+# python: "3.8"
+# os: ubuntu-latest
+# tox_env: "py38-lsof-numpy-pexpect"
+# use_coverage: true
+# - name: "ubuntu-py38-pluggy"
+# python: "3.8"
+# os: ubuntu-latest
+# tox_env: "py38-pluggymain-pylib-xdist"
+# - name: "ubuntu-py38-freeze"
+# python: "3.8"
+# os: ubuntu-latest
+# tox_env: "py38-freeze"
+# - name: "ubuntu-py39"
+# python: "3.9"
+# os: ubuntu-latest
+# tox_env: "py39-xdist"
+# - name: "ubuntu-py310"
+# python: "3.10"
+# os: ubuntu-latest
+# tox_env: "py310-xdist"
+# - name: "ubuntu-py311"
+# python: "3.11"
+# os: ubuntu-latest
+# tox_env: "py311"
+# use_coverage: true
+# - name: "ubuntu-py312"
+# python: "3.12"
+# os: ubuntu-latest
+# tox_env: "py312"
+# use_coverage: true
+# - name: "ubuntu-py313"
+# python: "3.13-dev"
+# os: ubuntu-latest
+# tox_env: "py313-pexpect"
+# use_coverage: true
+# - name: "ubuntu-pypy3"
+# python: "pypy-3.9"
+# os: ubuntu-latest
+# tox_env: "pypy3-xdist"
+
+# - name: "macos-py38"
+# python: "3.8"
+# os: macos-latest
+# tox_env: "py38-xdist"
+# - name: "macos-py39"
+# python: "3.9"
+# os: macos-latest
+# tox_env: "py39-xdist"
+# use_coverage: true
+# - name: "macos-py310"
+# python: "3.10"
+# os: macos-latest
+# tox_env: "py310-xdist"
+# - name: "macos-py312"
+# python: "3.12"
+# os: macos-latest
+# tox_env: "py312-xdist"
+# - name: "macos-py313"
+# python: "3.13-dev"
+# os: macos-latest
+# tox_env: "py313-xdist"
+
+# - name: "plugins"
+# python: "3.12"
+# os: ubuntu-latest
+# tox_env: "plugins"
+
+# - name: "doctesting"
+# python: "3.8"
+# os: ubuntu-latest
+# tox_env: "doctesting"
+# use_coverage: true
+
+# continue-on-error: >-
+# ${{
+# contains(
+# fromJSON(
+# '[
+# "windows-py38-pluggy",
+# "windows-py313",
+# "ubuntu-py38-pluggy",
+# "ubuntu-py38-freeze",
+# "ubuntu-py313",
+# "macos-py38",
+# "macos-py313"
+# ]'
+# ),
+# matrix.name
+# )
+# && true
+# || false
+# }}
+
+# steps:
+# - uses: actions/checkout@v4
+# with:
+# fetch-depth: 0
+# persist-credentials: false
+
+# - name: Download Package
+# uses: actions/download-artifact@v4
+# with:
+# name: Packages
+# path: dist
+
+# - name: Set up Python ${{ matrix.python }}
+# uses: actions/setup-python@v5
+# with:
+# python-version: ${{ matrix.python }}
+# check-latest: ${{ endsWith(matrix.python, '-dev') }}
+
+# - name: Install dependencies
+# run: |
+# python -m pip install --upgrade pip
+# pip install tox coverage
+
+# - name: Test without coverage
+# if: "! matrix.use_coverage"
+# shell: bash
+# run: tox run -e ${{ matrix.tox_env }} --installpkg `find dist/*.tar.gz`
+
+# - name: Test with coverage
+# if: "matrix.use_coverage"
+# shell: bash
+# run: tox run -e ${{ matrix.tox_env }}-coverage --installpkg `find dist/*.tar.gz`
+
+# - name: Generate coverage report
+# if: "matrix.use_coverage"
+# run: python -m coverage xml
+
+# - name: Upload coverage to Codecov
+# if: "matrix.use_coverage"
+# uses: codecov/codecov-action@v5
+# with:
+# fail_ci_if_error: false
+# files: ./coverage.xml
+# verbose: true
+
+# check: # This job does nothing and is only used for the branch protection
+# if: always()
+
+# needs:
+# - build
+
+# runs-on: ubuntu-latest
+
+# steps:
+# - name: Decide whether the needed jobs succeeded or failed
+# uses: re-actors/alls-green@223e4bb7a751b91f43eda76992bcfbf23b8b0302
+# with:
+# jobs: ${{ toJSON(needs) }}
diff --git a/.github/workflows/test_old_with_prioritization.yml b/.github/workflows/test_old_with_prioritization.yml
new file mode 100644
index 00000000000..3fcb44a4229
--- /dev/null
+++ b/.github/workflows/test_old_with_prioritization.yml
@@ -0,0 +1,637 @@
+name: test
+
+on:
+ push:
+ branches:
+ - main
+ - "[0-9]+.[0-9]+.x"
+ - pr-fails-simulation
+ tags:
+ - "[0-9]+.[0-9]+.[0-9]+"
+ - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
+ pull_request:
+ branches:
+ - main
+ - "[0-9]+.[0-9]+.x"
+ types:
+ - opened
+ - synchronize
+ - reopened
+ - ready_for_review
+
+env:
+ PYTEST_ADDOPTS: "--color=yes"
+
+# Cancel running jobs for the same workflow and branch.
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+# Set permissions at the job level.
+permissions: {}
+
+jobs:
+ package:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+ - name: Build and Check Package
+ uses: hynek/build-and-inspect-python-package@v2.12.0
+
+ build:
+ needs: [package]
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 45
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix:
+ name: [
+ "windows-py39-unittestextras",
+ "windows-py39-pluggy",
+ "windows-py39-xdist",
+ "windows-py310",
+ "windows-py311",
+ "windows-py312",
+ "windows-py313",
+ "ubuntu-py39-lsof-numpy-pexpect",
+ "ubuntu-py39-pluggy",
+ "ubuntu-py39-freeze",
+ "ubuntu-py39-xdist",
+ "ubuntu-py310-xdist",
+ "ubuntu-py311",
+ "ubuntu-py312",
+ "ubuntu-py313-pexpect",
+ "ubuntu-pypy3-xdist",
+ "macos-py39",
+ "macos-py310",
+ "macos-py312",
+ "macos-py313",
+ "doctesting",
+ "plugins",
+ ]
+ include:
+ - name: "windows-py39-unittestextras"
+ python: "3.9"
+ os: windows-latest
+ tox_env: "py39-unittestextras"
+ use_coverage: true
+ - name: "windows-py39-pluggy"
+ python: "3.9"
+ os: windows-latest
+ tox_env: "py39-pluggymain-pylib-xdist"
+ - name: "windows-py39-xdist"
+ python: "3.9"
+ os: windows-latest
+ tox_env: "py39-xdist"
+ - name: "windows-py310"
+ python: "3.10"
+ os: windows-latest
+ tox_env: "py310-xdist"
+ - name: "windows-py311"
+ python: "3.11"
+ os: windows-latest
+ tox_env: "py311"
+ - name: "windows-py312"
+ python: "3.12"
+ os: windows-latest
+ tox_env: "py312"
+ - name: "windows-py313"
+ python: "3.13"
+ os: windows-latest
+ tox_env: "py313"
+ - name: "ubuntu-py39-lsof-numpy-pexpect"
+ python: "3.9"
+ os: ubuntu-latest
+ tox_env: "py39-lsof-numpy-pexpect"
+ use_coverage: true
+ - name: "ubuntu-py39-pluggy"
+ python: "3.9"
+ os: ubuntu-latest
+ tox_env: "py39-pluggymain-pylib-xdist"
+ - name: "ubuntu-py39-freeze"
+ python: "3.9"
+ os: ubuntu-latest
+ tox_env: "py39-freeze"
+ - name: "ubuntu-py39-xdist"
+ python: "3.9"
+ os: ubuntu-latest
+ tox_env: "py39-xdist"
+ - name: "ubuntu-py310-xdist"
+ python: "3.10"
+ os: ubuntu-latest
+ tox_env: "py310-xdist"
+ - name: "ubuntu-py311"
+ python: "3.11"
+ os: ubuntu-latest
+ tox_env: "py311"
+ use_coverage: true
+ - name: "ubuntu-py312"
+ python: "3.12"
+ os: ubuntu-latest
+ tox_env: "py312"
+ use_coverage: true
+ - name: "ubuntu-py313-pexpect"
+ python: "3.13"
+ os: ubuntu-latest
+ tox_env: "py313-pexpect"
+ use_coverage: true
+ - name: "ubuntu-pypy3-xdist"
+ python: "pypy-3.9"
+ os: ubuntu-latest
+ tox_env: "pypy3-xdist"
+ - name: "macos-py39"
+ python: "3.9"
+ os: macos-latest
+ tox_env: "py39-xdist"
+ use_coverage: true
+ - name: "macos-py310"
+ python: "3.10"
+ os: macos-latest
+ tox_env: "py310-xdist"
+ - name: "macos-py312"
+ python: "3.12"
+ os: macos-latest
+ tox_env: "py312-xdist"
+ - name: "macos-py313"
+ python: "3.13"
+ os: macos-latest
+ tox_env: "py313-xdist"
+ - name: "plugins"
+ python: "3.12"
+ os: ubuntu-latest
+ tox_env: "plugins"
+ - name: "doctesting"
+ python: "3.9"
+ os: ubuntu-latest
+ tox_env: "doctesting"
+ use_coverage: true
+ continue-on-error: >-
+ ${{
+ contains(
+ fromJSON(
+ '[
+ "windows-py39-pluggy",
+ "windows-py313",
+ "ubuntu-py39-pluggy",
+ "ubuntu-py39-freeze",
+ "ubuntu-py313",
+ "macos-py39",
+ "macos-py313"
+ ]'
+ ),
+ matrix.name
+ )
+ && true
+ || false
+ }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Download Package
+ uses: actions/download-artifact@v4
+ with:
+ name: Packages
+ path: dist
+
+ - name: Set up Python ${{ matrix.python }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python }}
+ check-latest: ${{ endsWith(matrix.python, '-dev') }}
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install tox coverage pytest-json-report jq
+
+ - name: Get PR ID
+ shell: bash
+ if: github.event_name == 'pull_request'
+ run: echo "PR_ID=${{ github.event.number }}" >> $GITHUB_ENV
+
+ - name: Set Default Folder for Non-PR Runs
+ shell: bash
+ if: github.event_name != 'pull_request'
+ run: echo "PR_ID=main" >> $GITHUB_ENV
+
+ - name: Set Workflow ID
+ shell: bash
+ run: echo "WORKFLOW_ID=${{ matrix.name }}" >> $GITHUB_ENV
+
+ - name: Create Artifacts Directory
+ shell: bash
+ run: mkdir -p artifacts/pr-${PR_ID}/${WORKFLOW_ID}
+
+ - name: Check If Previous Artifacts Exist
+ id: check_artifacts
+ shell: bash
+ run: |
+ echo "Checking if previous test results exist for PR-${PR_ID}/${WORKFLOW_ID}..."
+ ARTIFACTS_RESPONSE=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+ -H "Accept: application/vnd.github.v3+json" \
+ "https://api.github.com/repos/${{ github.repository }}/actions/artifacts")
+
+ ARTIFACT_COUNT=$(echo "$ARTIFACTS_RESPONSE" | jq -r --arg PR "pr-${PR_ID}-${WORKFLOW_ID}-test-results" \
+ '[.artifacts[] | select(.name==$PR)] | length')
+
+ if [[ "$ARTIFACT_COUNT" -gt 0 ]]; then
+ echo "PREV_ARTIFACT_EXISTS=true" >> $GITHUB_ENV
+ else
+ echo "PREV_ARTIFACT_EXISTS=false" >> $GITHUB_ENV
+ fi
+
+ - name: Retrieve Previous Artifacts
+ if: env.PREV_ARTIFACT_EXISTS == 'true'
+ shell: bash
+ run: |
+ echo "Fetching previous test results for PR ${PR_ID}/${WORKFLOW_ID}..."
+ ARTIFACT_URL=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+ -H "Accept: application/vnd.github.v3+json" \
+ "https://api.github.com/repos/${{ github.repository }}/actions/artifacts" | \
+ jq -r --arg PR "pr-${PR_ID}-${WORKFLOW_ID}-test-results" \
+ '[.artifacts[] | select(.name==$PR)] | sort_by(.created_at) | reverse | .[0].archive_download_url')
+
+ if [[ -n "$ARTIFACT_URL" && "$ARTIFACT_URL" != "null" ]]; then
+ echo "Latest artifact found. Downloading..."
+ mkdir -p artifacts/pr-${PR_ID}/${WORKFLOW_ID}
+ curl -L -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+ -o artifacts/pr-${PR_ID}/${WORKFLOW_ID}/test-results.zip "$ARTIFACT_URL"
+ unzip -o artifacts/pr-${PR_ID}/${WORKFLOW_ID}/test-results.zip -d artifacts/pr-${PR_ID}/${WORKFLOW_ID}
+
+ echo "======================================="
+ echo "Previous Test Results for PR-${PR_ID}/${WORKFLOW_ID}:"
+ cat artifacts/pr-${PR_ID}/${WORKFLOW_ID}/test_results.json || echo "No previous test results found."
+ echo "======================================="
+ else
+ echo "No previous test results found for PR-${PR_ID}/${WORKFLOW_ID}. Running fresh tests."
+ fi
+
+ - name: Collect All Test Cases
+ shell: bash
+ run: |
+ mkdir -p artifacts/pr-${PR_ID}/${WORKFLOW_ID}
+ ALL_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/all_tests.txt"
+
+ # Use tox to collect all tests
+ tox -e ${{ matrix.tox_env }} --installpkg `find dist/*.tar.gz` -- --collect-only --quiet | grep -v "SKIP" | grep "::" > $ALL_TESTS_FILE || true
+
+ echo "Collected $(wc -l < $ALL_TESTS_FILE) test cases."
+
+ - name: Extract Failed Tests from Previous Run
+ shell: bash
+ run: |
+ PREV_RESULTS="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/test_results.json"
+ FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/failed_tests.txt"
+ ALL_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/all_tests.txt"
+ REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/remaining_tests.txt"
+
+ if [[ -f "$PREV_RESULTS" ]]; then
+ echo "Extracting failed test cases from previous run..."
+ cat $PREV_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_TESTS_FILE
+ else
+ echo "No previous test results found. Skipping extraction."
+ touch $FAILED_TESTS_FILE
+ fi
+
+ if [[ -s "$FAILED_TESTS_FILE" ]]; then
+ echo "Failed tests from the previous run:"
+ cat $FAILED_TESTS_FILE
+
+ # Identify remaining tests (all tests minus failed tests)
+ grep -v -F -f $FAILED_TESTS_FILE $ALL_TESTS_FILE > $REMAINING_TESTS_FILE || true
+ else
+ echo "No previously failed tests found."
+ cp $ALL_TESTS_FILE $REMAINING_TESTS_FILE
+ fi
+
+ - name: Pre-Check for Skipped Tests
+ shell: bash
+ run: |
+ FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/failed_tests.txt"
+ SKIPPED_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/skipped_tests.txt"
+
+ # Only run this check if we have previously failed tests
+ if [[ -s "$FAILED_TESTS_FILE" ]]; then
+ echo "Checking for skipped tests among previously failed tests..."
+ tox -e ${{ matrix.tox_env }} --installpkg `find dist/*.tar.gz` -- --collect-only -v $(cat $FAILED_TESTS_FILE) | grep "SKIP" | grep "::" | sed 's/.*SKIP //g' > $SKIPPED_TESTS_FILE || true
+
+ # Remove skipped tests from the failed tests list
+ if [[ -s "$SKIPPED_TESTS_FILE" ]]; then
+ echo "Removing skipped tests from the rerun list:"
+ cat $SKIPPED_TESTS_FILE
+ grep -v -F -f $SKIPPED_TESTS_FILE $FAILED_TESTS_FILE > "artifacts/pr-${PR_ID}/${WORKFLOW_ID}/filtered_failed_tests.txt"
+ mv "artifacts/pr-${PR_ID}/${WORKFLOW_ID}/filtered_failed_tests.txt" $FAILED_TESTS_FILE
+ else
+ echo "No skipped tests found among previously failed tests."
+ fi
+ fi
+
+ - name: Generate Failed Test Commands
+ shell: bash
+ run: |
+ FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/failed_tests.txt"
+
+ if [[ -s "$FAILED_TESTS_FILE" ]]; then
+ echo "Generating commands for previously failed tests..."
+ python scripts/generate_pytest_commands.py --input $FAILED_TESTS_FILE --output-dir artifacts --pr-id ${PR_ID} --workflow-id ${WORKFLOW_ID} --generate-script --batch-size 50 --tox-env ${{ matrix.tox_env }} --prefix failed
+ else
+ echo "No previously failed tests to generate commands for."
+ fi
+
+ - name: Run Previously Failed Tests First
+ shell: bash
+ run: |
+ FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/failed_tests.txt"
+ FAILED_SCRIPT="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/run_failed_tests.sh"
+
+ if [[ -s "$FAILED_TESTS_FILE" ]]; then
+ echo "Rerunning previously failed tests using tox env ${{ matrix.tox_env }}..."
+
+ if [[ -f "$FAILED_SCRIPT" ]]; then
+ chmod +x "$FAILED_SCRIPT"
+ if [[ "${{ matrix.use_coverage }}" == "true" ]]; then
+ # Use the coverage-enabled tox environment
+ sed -i 's/tox -e ${{ matrix.tox_env }}/tox -e ${{ matrix.tox_env }}-coverage/g' "$FAILED_SCRIPT"
+ fi
+ bash "$FAILED_SCRIPT"
+ else
+ echo "No failed test script generated."
+ fi
+ else
+ echo "No previously failed tests found."
+ fi
+
+ - name: Check If Any Tests Failed Again
+ shell: bash
+ run: |
+ TEMP_RESULTS="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/test_results.json"
+ FAILED_AGAIN_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/failed_again.txt"
+
+ if [[ -f "$TEMP_RESULTS" ]]; then
+ echo "Analyzing test results..."
+
+ # Extract failed tests (excluding skipped)
+ cat $TEMP_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_AGAIN_FILE || true
+
+ # Extract skipped tests for reporting
+ cat $TEMP_RESULTS | jq -r '.tests | map(select(.outcome == "skipped")) | .[].nodeid' > "artifacts/pr-${PR_ID}/${WORKFLOW_ID}/skipped_tests_report.txt" || true
+
+ # Report on skipped tests
+ if [[ -s "artifacts/pr-${PR_ID}/${WORKFLOW_ID}/skipped_tests_report.txt" ]]; then
+ echo "The following tests were skipped during execution:"
+ cat "artifacts/pr-${PR_ID}/${WORKFLOW_ID}/skipped_tests_report.txt"
+ fi
+ fi
+
+ if [[ -s "$FAILED_AGAIN_FILE" ]]; then
+ echo "Some tests failed again. Stopping execution."
+ echo "Failed tests:"
+ cat $FAILED_AGAIN_FILE
+ exit 1
+ fi
+
+ - name: Generate Commands for Remaining Tests
+ shell: bash
+ run: |
+ REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/remaining_tests.txt"
+
+ if [[ -s "$REMAINING_TESTS_FILE" ]]; then
+ echo "Generating commands for remaining tests..."
+ python scripts/generate_pytest_commands.py --input $REMAINING_TESTS_FILE --output-dir artifacts --pr-id ${PR_ID} --workflow-id ${WORKFLOW_ID} --generate-script --batch-size 50 --tox-env ${{ matrix.tox_env }}
+ else
+ echo "No remaining tests to generate commands for."
+ fi
+
+ - name: Run Remaining Test Cases
+ shell: bash
+ run: |
+ REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/remaining_tests.txt"
+ RUN_TESTS_SCRIPT="artifacts/pr-${PR_ID}/${WORKFLOW_ID}/run_tests.sh"
+
+ if [[ -s "$REMAINING_TESTS_FILE" ]]; then
+ echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..."
+
+ if [[ -f "$RUN_TESTS_SCRIPT" ]]; then
+ chmod +x "$RUN_TESTS_SCRIPT"
+ if [[ "${{ matrix.use_coverage }}" == "true" ]]; then
+ # Use the coverage-enabled tox environment
+ sed -i 's/tox -e ${{ matrix.tox_env }}/tox -e ${{ matrix.tox_env }}-coverage/g' "$RUN_TESTS_SCRIPT"
+ fi
+ bash "$RUN_TESTS_SCRIPT"
+
+ # Combine results after running tests
+ python scripts/generate_pytest_commands.py --combine-results --output-dir=artifacts --pr-id=${PR_ID} --workflow-id=${WORKFLOW_ID}
+ else
+ echo "No test script generated."
+ fi
+ else
+ echo "No remaining tests to run."
+ fi
+
+ - name: Generate coverage report
+ if: "matrix.use_coverage && !failure()"
+ shell: bash
+ run: |
+ if [[ -d ".coverage" || -f ".coverage" ]]; then
+ python -m coverage xml
+ else
+ echo "Looking for coverage data in tox environment..."
+ # Try to find and copy the coverage data from tox environment
+ TOX_ENV_DIR=".tox/${{ matrix.tox_env }}-coverage"
+ if [[ -d "$TOX_ENV_DIR" ]]; then
+ if [[ -d "$TOX_ENV_DIR/.coverage" || -f "$TOX_ENV_DIR/.coverage" ]]; then
+ cp -r "$TOX_ENV_DIR/.coverage" .
+ python -m coverage xml
+ else
+ echo "No coverage data found in tox environment."
+ # Create empty coverage file to prevent failure
+ echo '' > coverage.xml
+ fi
+ else
+ echo "Tox environment directory not found."
+ # Create empty coverage file to prevent failure
+ echo '' > coverage.xml
+ fi
+ fi
+ - name: Upload coverage to Codecov
+ if: "matrix.use_coverage && !failure()"
+ uses: codecov/codecov-action@v5
+ with:
+ fail_ci_if_error: false
+ files: ./coverage.xml
+ verbose: true
+
+ - name: Upload Test Results
+ uses: actions/upload-artifact@v4
+ with:
+ name: pr-${{ env.PR_ID }}-${{ env.WORKFLOW_ID }}-test-results
+ path: artifacts/pr-${{ env.PR_ID }}/${{ env.WORKFLOW_ID }}/test_results.json
+
+ retrieve-results:
+ needs: build
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ steps:
+ - name: Get PR ID
+ if: github.event_name == 'pull_request'
+ run: echo "PR_ID=${{ github.event.number }}" >> $GITHUB_ENV
+
+ - name: Set Default Folder for Non-PR Runs
+ if: github.event_name != 'pull_request'
+ run: echo "PR_ID=main" >> $GITHUB_ENV
+
+ - name: Check out code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.9"
+
+ - name: Install dependencies
+ shell: bash
+ run: |
+ python -m pip install --upgrade pip
+ pip install jq
+
+ - name: Create directory for results
+ run: mkdir -p retrieved-results
+
+ - name: Download all workflow test results
+ uses: actions/download-artifact@v4
+ with:
+ pattern: pr-${{ env.PR_ID }}-*-test-results
+ path: retrieved-results
+ merge-multiple: false
+
+ - name: Debug directory structure
+ shell: bash
+ run: |
+ echo "Debugging directory structure..."
+ find retrieved-results -type f | sort
+
+ - name: Extract zip files if needed
+ shell: bash
+ run: |
+ echo "Extracting downloaded artifacts..."
+ for zip_file in $(find retrieved-results -name "*.zip"); do
+ if [ -f "$zip_file" ]; then
+ workflow_dir=$(dirname "$zip_file")
+ echo "Extracting $zip_file to $workflow_dir"
+ unzip -o "$zip_file" -d "$workflow_dir"
+ fi
+ done
+
+ - name: Find test result files
+ shell: bash
+ run: |
+ echo "======================================="
+ echo "Downloaded artifacts for PR ${PR_ID}:"
+ find retrieved-results -type f -name "test_results*.json" | sort
+ echo "======================================="
+
+ - name: Combine test results
+ shell: bash
+ run: |
+ echo "Combining test results from all workflows..."
+
+ # Initialize combined results file
+ cat > retrieved-results/combined_results.json << EOF
+ {
+ "created": "$(date -Iseconds)",
+ "duration": 0,
+ "exitcode": 0,
+ "summary": {
+ "passed": 0,
+ "failed": 0,
+ "skipped": 0,
+ "xfailed": 0,
+ "xpassed": 0,
+ "error": 0,
+ "total": 0
+ },
+ "tests": [],
+ "collectors": [],
+ "warnings": []
+ }
+ EOF
+
+ # Find all test_results.json files
+ for result_file in $(find retrieved-results -type f -name "test_results*.json"); do
+ echo "Processing $result_file"
+
+ # Check if file is valid JSON
+ if ! jq empty "$result_file" 2>/dev/null; then
+ echo "Warning: $result_file is not valid JSON, skipping"
+ continue
+ fi
+
+ # Update summary counts
+ for metric in passed failed skipped xfailed xpassed error total; do
+ count=$(jq -r ".summary.$metric // 0" "$result_file")
+ current=$(jq -r ".summary.$metric" retrieved-results/combined_results.json)
+ new_count=$((current + count))
+ jq --arg metric "$metric" --argjson count "$new_count" '.summary[$metric] = $count' retrieved-results/combined_results.json > temp.json && mv temp.json retrieved-results/combined_results.json
+ done
+
+ # Add tests
+ jq -s '.[0].tests = (.[0].tests + (.[1].tests // [])); .[0]' retrieved-results/combined_results.json "$result_file" > temp.json && mv temp.json retrieved-results/combined_results.json
+
+ # Add duration
+ duration=$(jq -r ".duration // 0" "$result_file")
+ current_duration=$(jq -r ".duration" retrieved-results/combined_results.json)
+ new_duration=$(echo "$current_duration + $duration" | bc)
+ jq --argjson duration "$new_duration" '.duration = $duration' retrieved-results/combined_results.json > temp.json && mv temp.json retrieved-results/combined_results.json
+
+ # Update exitcode (non-zero takes precedence)
+ exitcode=$(jq -r ".exitcode // 0" "$result_file")
+ current_exitcode=$(jq -r ".exitcode" retrieved-results/combined_results.json)
+ if [ "$exitcode" -ne 0 ] && [ "$current_exitcode" -eq 0 ]; then
+ jq --argjson exitcode "$exitcode" '.exitcode = $exitcode' retrieved-results/combined_results.json > temp.json && mv temp.json retrieved-results/combined_results.json
+ fi
+ done
+
+ # Create a copy as test_results.json for backward compatibility
+ cp retrieved-results/combined_results.json retrieved-results/test_results.json
+
+ - name: Display Combined Test Results
+ shell: bash
+ run: |
+ echo "======================================="
+ echo "Combined Test Results from PR ${PR_ID}:"
+ echo "Summary:"
+ jq '.summary' retrieved-results/combined_results.json
+
+ echo "Failed Tests:"
+ jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' retrieved-results/combined_results.json || echo "No failed tests found."
+ echo "======================================="
+
+ - name: Upload Combined Results
+ uses: actions/upload-artifact@v4
+ with:
+ name: pr-${{ env.PR_ID }}-combined-test-results
+ path: retrieved-results/combined_results.json
+
+ check: # This job does nothing and is only used for the branch protection
+ if: always()
+ needs:
+ - build
+ runs-on: ubuntu-latest
+ steps:
+ - name: Decide whether the needed jobs succeeded or failed
+ uses: re-actors/alls-green@223e4bb7a751b91f43eda76992bcfbf23b8b0302
+ with:
+ jobs: ${{ toJSON(needs) }}
diff --git a/changelog/13403.bugfix.rst b/changelog/13403.bugfix.rst
new file mode 100644
index 00000000000..132cbfe0010
--- /dev/null
+++ b/changelog/13403.bugfix.rst
@@ -0,0 +1 @@
+Disable assertion rewriting of external modules
diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py
index 6f550bdfdc8..2fcabe27eb2 100644
--- a/scripts/generate_pytest_commands.py
+++ b/scripts/generate_pytest_commands.py
@@ -134,10 +134,10 @@ def create_test_batch_json(test_list, output_dir, pr_id, workflow_id, batch_size
current_size = 0
for module, tests in test_modules.items():
# Use smaller batch size for modules with many tests
- if len(tests) > 51:
+ if len(tests) > batch_size+5:
# Split large modules into smaller batches of 5 tests each
- for i in range(0, len(tests), 10):
- batches.append(tests[i:i+10])
+ for i in range(0, len(tests), 20):
+ batches.append(tests[i:i+20])
else:
# For smaller modules, keep using the module-based approach
if current_size + len(tests) > batch_size and current_batch:
diff --git a/src/_pytest/assertion/rewrite.py b/src/_pytest/assertion/rewrite.py
index 2e606d1903a..5cdf7f0f5bc 100644
--- a/src/_pytest/assertion/rewrite.py
+++ b/src/_pytest/assertion/rewrite.py
@@ -218,6 +218,10 @@ def _early_rewrite_bailout(self, name: str, state: AssertionState) -> bool:
if fnmatch_ex(pat, path):
return False
+ root_path = self._get_root_path()
+ if not path.is_relative_to(root_path):
+ return True
+
if self._is_marked_for_rewrite(name, state):
return False
@@ -238,6 +242,10 @@ def _should_rewrite(self, name: str, fn: str, state: AssertionState) -> bool:
# modules not passed explicitly on the command line are only
# rewritten if they match the naming convention for test files
fn_path = PurePath(fn)
+ root_path = self._get_root_path()
+ if not fn_path.is_relative_to(root_path):
+ return False
+
for pat in self.fnpats:
if fnmatch_ex(pat, fn_path):
state.trace(f"matched test file {fn!r}")
@@ -245,6 +253,14 @@ def _should_rewrite(self, name: str, fn: str, state: AssertionState) -> bool:
return self._is_marked_for_rewrite(name, state)
+ @staticmethod
+ def _get_root_path():
+ try:
+ root_path = os.getcwd()
+ return root_path
+ except FileNotFoundError:
+ return os.path.dirname(os.path.abspath(sys.argv[0]))
+
def _is_marked_for_rewrite(self, name: str, state: AssertionState) -> bool:
try:
return self._marked_for_rewrite_cache[name]
diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py
index 02d1c3e52ff..22028d5db39 100644
--- a/testing/test_assertrewrite.py
+++ b/testing/test_assertrewrite.py
@@ -22,6 +22,8 @@
from unittest import mock
import zipfile
+from _pytest.monkeypatch import MonkeyPatch
+
import _pytest._code
from _pytest._io.saferepr import DEFAULT_REPR_MAX_SIZE
from _pytest.assertion import util
@@ -1955,6 +1957,27 @@ def test_simple_failure():
assert hook.find_spec("file") is not None
assert self.find_spec_calls == ["file"]
+
+ def test_assert_excluded_rootpath(
+ self, pytester: Pytester, hook: AssertionRewritingHook, monkeypatch
+ ) -> None:
+ """
+ If test files contained outside rootdir, then skip them
+ """
+ pytester.makepyfile(
+ **{
+ "file.py": """\
+ def test_simple_failure():
+ assert 1 + 1 == 3
+ """
+ }
+ )
+ root_path= "{0}/tests".format(os.getcwd())
+ monkeypatch.setattr("os.getcwd", lambda: root_path)
+ with mock.patch.object(hook, "fnpats", ["*.py"]):
+ assert hook.find_spec("file") is None
+
+
@pytest.mark.skipif(
sys.platform.startswith("win32"), reason="cannot remove cwd on Windows"
)