diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e19ee67ccf0..4a5e9c9ca82 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,6 @@ jobs: runs-on: ${{ matrix.os }} permissions: contents: read - strategy: fail-fast: false matrix: @@ -46,38 +45,31 @@ jobs: "windows-py312", "windows-py313" ] - include: - name: "windows-py39-unittestextras" python: "3.9" os: windows-latest tox_env: "py39-unittestextras" - - name: "windows-py39-pluggy" python: "3.9" os: windows-latest tox_env: "py39-pluggymain-pylib-xdist" - - name: "windows-py39-xdist" python: "3.9" os: windows-latest tox_env: "py39-xdist" - - name: "windows-py310" python: "3.10" os: windows-latest tox_env: "py310-xdist" - - name: "windows-py311" python: "3.11" os: windows-latest tox_env: "py311" - - name: "windows-py312" python: "3.12" os: windows-latest tox_env: "py312" - - name: "windows-py313" python: "3.13" os: windows-latest @@ -107,18 +99,25 @@ jobs: if: github.event_name != 'pull_request' run: echo "PR_ID=main" >> $GITHUB_ENV + # Create a unique artifact directory for this matrix job + - name: Set Artifact Directory + shell: bash + run: | + export ARTIFACT_DIR="artifacts/pr-${PR_ID}/${{ matrix.name }}" + echo "ARTIFACT_DIR=${ARTIFACT_DIR}" >> $GITHUB_ENV + mkdir -p "$ARTIFACT_DIR" + + # Check if previous artifacts exist for THIS matrix job - name: Check If Previous Artifacts Exist id: check_artifacts shell: bash run: | - echo "Checking if previous test results exist for PR-${PR_ID}..." + echo "Checking for previous test results for PR-${PR_ID}-${{ matrix.name }}..." ARTIFACTS_RESPONSE=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/${{ github.repository }}/actions/artifacts") - - ARTIFACT_COUNT=$(echo "$ARTIFACTS_RESPONSE" | jq -r --arg PR "pr-${PR_ID}-test-results" \ - '[.artifacts[] | select(.name==$PR)] | length') - + ARTIFACT_COUNT=$(echo "$ARTIFACTS_RESPONSE" | jq -r --arg ART "pr-${PR_ID}-${{ matrix.name }}-test-results" \ + '[.artifacts[] | select(.name==$ART)] | length') if [[ "$ARTIFACT_COUNT" -gt 0 ]]; then echo "PREV_ARTIFACT_EXISTS=true" >> $GITHUB_ENV else @@ -129,137 +128,116 @@ jobs: if: env.PREV_ARTIFACT_EXISTS == 'true' shell: bash run: | - echo "Fetching previous test results for PR ${PR_ID}..." - + echo "Fetching previous test results for PR ${PR_ID}-${{ matrix.name }}..." ARTIFACT_URL=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/${{ github.repository }}/actions/artifacts" | \ - jq -r --arg PR "pr-${PR_ID}-test-results" \ - '[.artifacts[] | select(.name==$PR)] | sort_by(.created_at) | reverse | .[0].archive_download_url') - + jq -r --arg ART "pr-${PR_ID}-${{ matrix.name }}-test-results" \ + '[.artifacts[] | select(.name==$ART)] | sort_by(.created_at) | reverse | .[0].archive_download_url') if [[ -n "$ARTIFACT_URL" && "$ARTIFACT_URL" != "null" ]]; then - echo "Latest artifact found. Downloading..." - mkdir -p artifacts/pr-${PR_ID} + echo "Previous artifact found. Downloading into $ARTIFACT_DIR..." curl -L -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -o artifacts/pr-${PR_ID}/test-results.zip "$ARTIFACT_URL" - unzip -o artifacts/pr-${PR_ID}/test-results.zip -d artifacts/pr-${PR_ID} - - echo "=======================================" - echo "Previous Test Results for PR-${PR_ID}:" - cat artifacts/pr-${PR_ID}/test_results.json || echo "No previous test results found." - echo "=======================================" + -o "$ARTIFACT_DIR/test-results.zip" "$ARTIFACT_URL" + unzip -o "$ARTIFACT_DIR/test-results.zip" -d "$ARTIFACT_DIR" else - echo "No previous test results found for PR-${PR_ID}. Running fresh tests." + echo "No previous artifact for PR-${PR_ID}-${{ matrix.name }}. Proceeding with fresh tests." fi - name: Extract Failed and Passed Tests from Previous Run shell: bash run: | - mkdir -p artifacts/pr-${PR_ID} - PREV_RESULTS="artifacts/pr-${PR_ID}/test_results.json" - FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt" - ALL_TESTS_FILE="artifacts/pr-${PR_ID}/all_tests.txt" - REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt" - - # Use tox to collect all tests - tox -e ${{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true - + mkdir -p "$ARTIFACT_DIR" + PREV_RESULTS="$ARTIFACT_DIR/test_results.json" + FAILED_TESTS_FILE="$ARTIFACT_DIR/failed_tests.txt" + ALL_TESTS_FILE="$ARTIFACT_DIR/all_tests.txt" + REMAINING_TESTS_FILE="$ARTIFACT_DIR/remaining_tests.txt" + + # Use tox to collect all tests + tox -e ${{ matrix.tox_env }} -- --collect-only --verbose | grep "::" | grep -v "\[SKIPPED\]" > "$ALL_TESTS_FILE" || true + if [[ -f "$PREV_RESULTS" ]]; then - echo "Extracting failed test cases from previous run..." - cat $PREV_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_TESTS_FILE + echo "Extracting failed tests from previous run..." + cat "$PREV_RESULTS" | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > "$FAILED_TESTS_FILE" else echo "No previous test results found. Skipping extraction." - touch $FAILED_TESTS_FILE + touch "$FAILED_TESTS_FILE" fi - + if [[ -s "$FAILED_TESTS_FILE" ]]; then - echo "Failed tests from the previous run:" - cat $FAILED_TESTS_FILE + echo "Failed tests found:" + cat "$FAILED_TESTS_FILE" else - echo "No previously failed tests found." + echo "No previously failed tests." fi - - name: Run Previously Failed Tests First shell: bash run: | - FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt" - + FAILED_TESTS_FILE="$ARTIFACT_DIR/failed_tests.txt" if [[ -s "$FAILED_TESTS_FILE" ]]; then echo "Rerunning previously failed tests using tox env ${{ matrix.tox_env }}..." - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report --json-report-file=artifacts/pr-${PR_ID}/temp_test_results.json $(cat $FAILED_TESTS_FILE) || true + tox -e ${{ matrix.tox_env }} -- --tb=short --json-report --json-report-file="$ARTIFACT_DIR/temp_test_results.json" $(cat "$FAILED_TESTS_FILE") || true else - echo "No previously failed tests found." + echo "No previously failed tests to run." fi - name: Check If Any Tests Failed Again shell: bash run: | - TEMP_RESULTS="artifacts/pr-${PR_ID}/temp_test_results.json" - FAILED_AGAIN_FILE="artifacts/pr-${PR_ID}/failed_again.txt" - + TEMP_RESULTS="$ARTIFACT_DIR/temp_test_results.json" + FAILED_AGAIN_FILE="$ARTIFACT_DIR/failed_again.txt" if [[ -f "$TEMP_RESULTS" ]]; then - echo "Checking if any tests failed again..." - cat $TEMP_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_AGAIN_FILE + echo "Checking if any tests failed again..." + cat "$TEMP_RESULTS" | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > "$FAILED_AGAIN_FILE" fi - if [[ -s "$FAILED_AGAIN_FILE" ]]; then - echo "Some tests failed again. Stopping execution." - exit 1 + echo "Some tests failed again. Exiting with error." + exit 1 fi - name: Identify Remaining Untested Test Cases shell: bash run: | - FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt" - ALL_TESTS_FILE="artifacts/pr-${PR_ID}/all_tests.txt" - REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt" - - echo "Finding remaining tests to run..." - grep -v -F -f $FAILED_TESTS_FILE $ALL_TESTS_FILE > $REMAINING_TESTS_FILE || true - + FAILED_TESTS_FILE="$ARTIFACT_DIR/failed_tests.txt" + ALL_TESTS_FILE="$ARTIFACT_DIR/all_tests.txt" + REMAINING_TESTS_FILE="$ARTIFACT_DIR/remaining_tests.txt" + echo "Identifying remaining tests to run..." + grep -v -F -f "$FAILED_TESTS_FILE" "$ALL_TESTS_FILE" > "$REMAINING_TESTS_FILE" || true if [[ -s "$REMAINING_TESTS_FILE" ]]; then - echo "Remaining tests to run:" - cat $REMAINING_TESTS_FILE + echo "Remaining tests to run:" + cat "$REMAINING_TESTS_FILE" else - echo "No remaining tests to run." + echo "No remaining tests to run." fi - + - name: Run Remaining Test Cases shell: bash run: | - REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt" - + REMAINING_TESTS_FILE="$ARTIFACT_DIR/remaining_tests.txt" if [[ -s "$REMAINING_TESTS_FILE" ]]; then - echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - - # 1. Split the test list into chunks of 300 lines each (adjust as needed). - CHUNK_SIZE=300 - split -l $CHUNK_SIZE $REMAINING_TESTS_FILE chunk_ - - i=1 - for chunk in chunk_*; do - echo "Running chunk #$i with $(wc -l < "$chunk") tests" - - # 2. Pass those tests as arguments to tox in smaller batches - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ - --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ - $(cat "$chunk") || true - - ((i++)) - done + echo "Running remaining tests using tox env ${{ matrix.tox_env }}..." + CHUNK_SIZE=300 + split -l $CHUNK_SIZE "$REMAINING_TESTS_FILE" chunk_ + i=1 + for chunk in chunk_*; do + echo "Running chunk #$i with $(wc -l < "$chunk") tests..." + tox -e ${{ matrix.tox_env }} -- --tb=short --json-report --json-report-file="$ARTIFACT_DIR/test_results.json" $(cat "$chunk") || true + ((i++)) + done else - echo "No remaining tests to run." + echo "No remaining tests to run." fi - - name: Upload New Test Results uses: actions/upload-artifact@v4 with: - name: pr-${{ env.PR_ID }}-test-results - path: artifacts/pr-${{ env.PR_ID }}/test_results.json + name: pr-${{ env.PR_ID }}-${{ matrix.name }}-test-results + path: ${{ env.ARTIFACT_DIR }}/**/* + + + retrieve-results: needs: run-tests runs-on: ubuntu-latest @@ -272,16 +250,59 @@ jobs: if: github.event_name != 'pull_request' run: echo "PR_ID=main" >> $GITHUB_ENV - - name: Download Test Results + # Download artifact for windows-py39-unittestextras + - name: Download artifact for windows-py39-unittestextras + uses: actions/download-artifact@v4 + with: + name: pr-${{ env.PR_ID }}-windows-py39-unittestextras-test-results + path: retrieved-results/windows-py39-unittestextras + + # Download artifact for windows-py39-pluggy + - name: Download artifact for windows-py39-pluggy + uses: actions/download-artifact@v4 + with: + name: pr-${{ env.PR_ID }}-windows-py39-pluggy-test-results + path: retrieved-results/windows-py39-pluggy + + # Download artifact for windows-py39-xdist + - name: Download artifact for windows-py39-xdist + uses: actions/download-artifact@v4 + with: + name: pr-${{ env.PR_ID }}-windows-py39-xdist-test-results + path: retrieved-results/windows-py39-xdist + + # Download artifact for windows-py310 + - name: Download artifact for windows-py310 uses: actions/download-artifact@v4 with: - name: pr-${{ env.PR_ID }}-test-results - path: retrieved-results + name: pr-${{ env.PR_ID }}-windows-py310-test-results + path: retrieved-results/windows-py310 + + # Download artifact for windows-py311 + - name: Download artifact for windows-py311 + uses: actions/download-artifact@v4 + with: + name: pr-${{ env.PR_ID }}-windows-py311-test-results + path: retrieved-results/windows-py311 + + # Download artifact for windows-py312 + - name: Download artifact for windows-py312 + uses: actions/download-artifact@v4 + with: + name: pr-${{ env.PR_ID }}-windows-py312-test-results + path: retrieved-results/windows-py312 + + # Download artifact for windows-py313 + - name: Download artifact for windows-py313 + uses: actions/download-artifact@v4 + with: + name: pr-${{ env.PR_ID }}-windows-py313-test-results + path: retrieved-results/windows-py313 - name: Display Retrieved Test Results - shell: bash run: | echo "=======================================" - echo "Retrieved Test Results from PR ${PR_ID}:" - cat retrieved-results/test_results.json + echo "Retrieved Test Results (all configurations):" + find retrieved-results -type f -name "test_results.json" -exec cat {} \; echo "=======================================" +