From f2b78979075c02fb9c3d160abf16e7ed9a15cce8 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 15:45:25 -0400 Subject: [PATCH 01/35] adding jobs to skip test cases and check for skipped test cases in the next run --- .github/workflows/test.yml | 41 +++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e19ee67ccf0..fe1a118d5a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -162,7 +162,8 @@ jobs: REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt" # Use tox to collect all tests - tox -e ${{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true + # tox -e $#{{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true + tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep "::" > $ALL_TESTS_FILE || true if [[ -f "$PREV_RESULTS" ]]; then echo "Extracting failed test cases from previous run..." @@ -178,7 +179,28 @@ jobs: else echo "No previously failed tests found." fi - + + - name: Pre-Check for Previously Failed Tests + shell: bash + run: | + FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt" + SKIPPED_TESTS_FILE="artifacts/pr-${PR_ID}/skipped_tests.txt" + + # Only run this check if we have previously failed tests + if [[ -s "$FAILED_TESTS_FILE" ]]; then + echo "Checking for skipped tests among previously failed tests..." + tox -e ${{ matrix.tox_env }} -- --collect-only -v $(cat $FAILED_TESTS_FILE) | grep "SKIP" | grep "::" | sed 's/.*SKIP //g' > $SKIPPED_TESTS_FILE + + # Remove skipped tests from the failed tests list + if [[ -s "$SKIPPED_TESTS_FILE" ]]; then + echo "Removing skipped tests from the rerun list:" + cat $SKIPPED_TESTS_FILE + grep -v -F -f $SKIPPED_TESTS_FILE $FAILED_TESTS_FILE > "artifacts/pr-${PR_ID}/filtered_failed_tests.txt" + mv "artifacts/pr-${PR_ID}/filtered_failed_tests.txt" $FAILED_TESTS_FILE + else + echo "No skipped tests found among previously failed tests." + fi + fi - name: Run Previously Failed Tests First shell: bash @@ -197,12 +219,21 @@ jobs: run: | TEMP_RESULTS="artifacts/pr-${PR_ID}/temp_test_results.json" FAILED_AGAIN_FILE="artifacts/pr-${PR_ID}/failed_again.txt" - + if [[ -f "$TEMP_RESULTS" ]]; then - echo "Checking if any tests failed again..." + echo "Analyzing test results..." + # Extract failed tests (excluding skipped) cat $TEMP_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_AGAIN_FILE + # Extract skipped tests for reporting + cat $TEMP_RESULTS | jq -r '.tests | map(select(.outcome == "skipped")) | .[].nodeid' > "artifacts/pr-${PR_ID}/skipped_tests_report.txt" + + # Report on skipped tests + if [[ -s "artifacts/pr-${PR_ID}/skipped_tests_report.txt" ]]; then + echo "The following tests were skipped during execution:" + cat "artifacts/pr-${PR_ID}/skipped_tests_report.txt" + fi fi - + if [[ -s "$FAILED_AGAIN_FILE" ]]; then echo "Some tests failed again. Stopping execution." exit 1 From 2a829f7464d52cb726eac554c74c73d4af41b767 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 15:51:12 -0400 Subject: [PATCH 02/35] removing hardcoded version --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fe1a118d5a1..b16fd1fd6e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,8 +18,8 @@ on: env: PYTEST_ADDOPTS: "--color=yes" - SETUPTOOLS_SCM_PRETEND_VERSION: "7.3.1.dev0" - SETUPTOOLS_SCM_NO_LOCAL_VERSION: "1" + # SETUPTOOLS_SCM_PRETEND_VERSION: "7.3.1.dev0" + # SETUPTOOLS_SCM_NO_LOCAL_VERSION: "1" concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 9e2b62d2c1d0f9a318c0bd1cb8f0a0335200561f Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 15:55:58 -0400 Subject: [PATCH 03/35] removing the part running test cases in chunks --- .github/workflows/test.yml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b16fd1fd6e7..3fe0fd16f49 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -260,28 +260,18 @@ jobs: shell: bash run: | REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt" - + if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - - # 1. Split the test list into chunks of 300 lines each (adjust as needed). - CHUNK_SIZE=300 - split -l $CHUNK_SIZE $REMAINING_TESTS_FILE chunk_ - - i=1 - for chunk in chunk_*; do - echo "Running chunk #$i with $(wc -l < "$chunk") tests" - - # 2. Pass those tests as arguments to tox in smaller batches - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ - --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ - $(cat "$chunk") || true - - ((i++)) - done + + # Run all tests at once instead of in chunks + tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ + --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ + $(cat "$REMAINING_TESTS_FILE") || true else echo "No remaining tests to run." fi + - name: Upload New Test Results From 230f57f428046e8a585639dc53d121371f5db5f6 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 16:15:10 -0400 Subject: [PATCH 04/35] manually installing pytest-json-report --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3fe0fd16f49..eb270d7ca5f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -95,15 +95,18 @@ jobs: python-version: ${{ matrix.python }} - name: Install dependencies + shell: bash run: | python -m pip install --upgrade pip pip install tox pytest-json-report jq - name: Get PR ID + shell: bash if: github.event_name == 'pull_request' run: echo "PR_ID=${{ github.event.number }}" >> $GITHUB_ENV - name: Set Default Folder for Non-PR Runs + shell: bash if: github.event_name != 'pull_request' run: echo "PR_ID=main" >> $GITHUB_ENV @@ -209,7 +212,7 @@ jobs: if [[ -s "$FAILED_TESTS_FILE" ]]; then echo "Rerunning previously failed tests using tox env ${{ matrix.tox_env }}..." - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report --json-report-file=artifacts/pr-${PR_ID}/temp_test_results.json $(cat $FAILED_TESTS_FILE) || true + tox -e ${{ matrix.tox_env }} --installpkg=pytest-json-report -- --tb=short --json-report --json-report-file=artifacts/pr-${PR_ID}/temp_test_results.json $(cat $FAILED_TESTS_FILE) || true else echo "No previously failed tests found." fi @@ -265,7 +268,7 @@ jobs: echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." # Run all tests at once instead of in chunks - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ + tox -e ${{ matrix.tox_env }} --installpkg=pytest-json-report -- --tb=short --json-report \ --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ $(cat "$REMAINING_TESTS_FILE") || true else From 4bb240e15ba9f6347f26723d5d71ee147e8ea832 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 16:25:31 -0400 Subject: [PATCH 05/35] adding pytest-json-report to tox.ini --- .github/workflows/test.yml | 4 ++-- tox.ini | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb270d7ca5f..b850cf7752d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -212,7 +212,7 @@ jobs: if [[ -s "$FAILED_TESTS_FILE" ]]; then echo "Rerunning previously failed tests using tox env ${{ matrix.tox_env }}..." - tox -e ${{ matrix.tox_env }} --installpkg=pytest-json-report -- --tb=short --json-report --json-report-file=artifacts/pr-${PR_ID}/temp_test_results.json $(cat $FAILED_TESTS_FILE) || true + tox -e ${{ matrix.tox_env }} -- --tb=short --json-report --json-report-file=artifacts/pr-${PR_ID}/temp_test_results.json $(cat $FAILED_TESTS_FILE) || true else echo "No previously failed tests found." fi @@ -268,7 +268,7 @@ jobs: echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." # Run all tests at once instead of in chunks - tox -e ${{ matrix.tox_env }} --installpkg=pytest-json-report -- --tb=short --json-report \ + tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ $(cat "$REMAINING_TESTS_FILE") || true else diff --git a/tox.ini b/tox.ini index 2f510915be1..1a28816b45f 100644 --- a/tox.ini +++ b/tox.ini @@ -73,6 +73,7 @@ deps = unittestextras: twisted unittestextras: asynctest xdist: pytest-xdist + pytest-json-report {env:_PYTEST_TOX_EXTRA_DEP:} [testenv:linting] From 69e3abedc29d3362b8d3d5a6c1d8f287a6644546 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 16:45:00 -0400 Subject: [PATCH 06/35] passing test cases as functions --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b850cf7752d..ee12f7183b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,8 +166,8 @@ jobs: # Use tox to collect all tests # tox -e $#{{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true - tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep "::" > $ALL_TESTS_FILE || true - + tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep "::" | sed 's//\1/' > $ALL_TESTS_FILE || true + if [[ -f "$PREV_RESULTS" ]]; then echo "Extracting failed test cases from previous run..." cat $PREV_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_TESTS_FILE @@ -193,7 +193,7 @@ jobs: if [[ -s "$FAILED_TESTS_FILE" ]]; then echo "Checking for skipped tests among previously failed tests..." tox -e ${{ matrix.tox_env }} -- --collect-only -v $(cat $FAILED_TESTS_FILE) | grep "SKIP" | grep "::" | sed 's/.*SKIP //g' > $SKIPPED_TESTS_FILE - + # Remove skipped tests from the failed tests list if [[ -s "$SKIPPED_TESTS_FILE" ]]; then echo "Removing skipped tests from the rerun list:" From a2a48ee0426163d1d12202e402ab1a360e49d4ac Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 16:52:28 -0400 Subject: [PATCH 07/35] modify your test collection command to preserve the full test paths --- .github/workflows/test.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ee12f7183b1..4d84b3cb917 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,8 +166,7 @@ jobs: # Use tox to collect all tests # tox -e $#{{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true - tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep "::" | sed 's//\1/' > $ALL_TESTS_FILE || true - + tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep "::" | sed -E 's/.*collected (.*)$/\1/' > $ALL_TESTS_FILE || true if [[ -f "$PREV_RESULTS" ]]; then echo "Extracting failed test cases from previous run..." cat $PREV_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_TESTS_FILE @@ -268,9 +267,8 @@ jobs: echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." # Run all tests at once instead of in chunks - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ - --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ - $(cat "$REMAINING_TESTS_FILE") || true + tox -e ${{ matrix.tox_env }} -- --tb=short --json-report --json-report-file=artifacts/pr-${PR_ID}/test_results.json -v --from-file="$REMAINING_TESTS_FILE" || true + else echo "No remaining tests to run." fi From 2e0c20c55349b7e149332db8eedb193d872b570e Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 16:56:12 -0400 Subject: [PATCH 08/35] modify the run command to read the file and pass the tests directly: --- .github/workflows/test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d84b3cb917..43745a2bdbb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -266,9 +266,10 @@ jobs: if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - # Run all tests at once instead of in chunks - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report --json-report-file=artifacts/pr-${PR_ID}/test_results.json -v --from-file="$REMAINING_TESTS_FILE" || true - + # Read the file and pass tests as arguments + tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ + --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ + -v $(cat "$REMAINING_TESTS_FILE") || true else echo "No remaining tests to run." fi From 14523287dc3dfffacd2bf63dda5929cf89f32472 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 17:02:15 -0400 Subject: [PATCH 09/35] modify the "Run Remaining Test Cases" step to add the -v flag and ensure the test identifiers are properly formatted --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43745a2bdbb..72b7dcb39c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -266,10 +266,13 @@ jobs: if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - # Read the file and pass tests as arguments + # Process the file to remove wrappers if they exist + cat "$REMAINING_TESTS_FILE" | sed 's//\1/' > "artifacts/pr-${PR_ID}/clean_tests.txt" + + # Run all tests at once with clean test identifiers tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ - -v $(cat "$REMAINING_TESTS_FILE") || true + -v $(cat "artifacts/pr-${PR_ID}/clean_tests.txt") || true else echo "No remaining tests to run." fi From 5047fa2e020cd13dfa8fd6ea1fed5568befb2604 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 17:10:40 -0400 Subject: [PATCH 10/35] correcting the issue: test names are being passed without their module paths, which prevents pytest from locating the test files. --- .github/workflows/test.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 72b7dcb39c0..2faa0ab505f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,7 +166,8 @@ jobs: # Use tox to collect all tests # tox -e $#{{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true - tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep "::" | sed -E 's/.*collected (.*)$/\1/' > $ALL_TESTS_FILE || true + tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | sed -E 's/.*$/\1/' > $ALL_TESTS_FILE || true + if [[ -f "$PREV_RESULTS" ]]; then echo "Extracting failed test cases from previous run..." cat $PREV_RESULTS | jq -r '.tests | map(select(.outcome == "failed")) | .[].nodeid' > $FAILED_TESTS_FILE @@ -266,17 +267,13 @@ jobs: if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - # Process the file to remove wrappers if they exist - cat "$REMAINING_TESTS_FILE" | sed 's//\1/' > "artifacts/pr-${PR_ID}/clean_tests.txt" - - # Run all tests at once with clean test identifiers + # Run all tests at once tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ - -v $(cat "artifacts/pr-${PR_ID}/clean_tests.txt") || true + -v $(cat "$REMAINING_TESTS_FILE") || true else echo "No remaining tests to run." fi - - name: Upload New Test Results From 70813a1b5b30494614abe0cf03d5a2c5d5010897 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 17:17:17 -0400 Subject: [PATCH 11/35] creating temp test file for running pytest command --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2faa0ab505f..a2b10c13295 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -267,10 +267,13 @@ jobs: if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - # Run all tests at once + # Create a temporary file with test names + cat "$REMAINING_TESTS_FILE" > temp_tests.txt + + # Run tests reading from the file tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ - -v $(cat "$REMAINING_TESTS_FILE") || true + -v $(< temp_tests.txt) || true else echo "No remaining tests to run." fi From 07b3e0b6e116510fcad03a2a3a62ec3cfcbce29c Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 17:23:12 -0400 Subject: [PATCH 12/35] running in batches --- .github/workflows/test.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a2b10c13295..e249c65302f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -267,13 +267,18 @@ jobs: if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - # Create a temporary file with test names - cat "$REMAINING_TESTS_FILE" > temp_tests.txt + # Split tests into batches of 50 tests each + split -l 50 "$REMAINING_TESTS_FILE" batch_ - # Run tests reading from the file - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ - --json-report-file=artifacts/pr-${PR_ID}/test_results.json \ - -v $(< temp_tests.txt) || true + # Run each batch + for batch in batch_*; do + echo "Running batch $batch..." + tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ + --json-report-file=artifacts/pr-${PR_ID}/test_results_${batch}.json \ + -v $(cat "$batch") || true + done + + # Combine results if needed else echo "No remaining tests to run." fi From 9f7b64e16a75fd7676175454bc3625eb4f805a63 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 17:46:37 -0400 Subject: [PATCH 13/35] Generate properly formatted JSON files for each batch of tests Create bash scripts with the correct tox commands Run the tests in manageable batches to avoid "Argument list too long" errors Properly handle both failed and remaining tests Upload all relevant artifacts for future runs This approach leverages generate_pytest_commands.py script to create structured JSON files and executable bash scripts, making your test workflow more robust and maintainable. --- .github/workflows/test.yml | 48 ++++++---- scripts/generate_pytest_commands.py | 134 ++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 scripts/generate_pytest_commands.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e249c65302f..2845b84bf36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -205,17 +205,33 @@ jobs: fi fi + - name: Generate Failed Test Commands + shell: bash + run: | + FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt" + + if [[ -s "$FAILED_TESTS_FILE" ]]; then + python scripts/generate_pytest_commands.py --input ${FAILED_TESTS_FILE} --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 --prefix failed + fi + - name: Run Previously Failed Tests First shell: bash run: | FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt" - + if [[ -s "$FAILED_TESTS_FILE" ]]; then echo "Rerunning previously failed tests using tox env ${{ matrix.tox_env }}..." - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report --json-report-file=artifacts/pr-${PR_ID}/temp_test_results.json $(cat $FAILED_TESTS_FILE) || true + + if [[ -f "artifacts/pr-${PR_ID}/run_failed_tests.sh" ]]; then + chmod +x artifacts/pr-${PR_ID}/run_failed_tests.sh + bash artifacts/pr-${PR_ID}/run_failed_tests.sh + else + echo "No failed test script generated." + fi else echo "No previously failed tests found." fi + - name: Check If Any Tests Failed Again shell: bash @@ -259,6 +275,11 @@ jobs: echo "No remaining tests to run." fi + - name: Generate Test Commands + shell: bash + run: | + python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 + - name: Run Remaining Test Cases shell: bash run: | @@ -267,18 +288,12 @@ jobs: if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - # Split tests into batches of 50 tests each - split -l 50 "$REMAINING_TESTS_FILE" batch_ - - # Run each batch - for batch in batch_*; do - echo "Running batch $batch..." - tox -e ${{ matrix.tox_env }} -- --tb=short --json-report \ - --json-report-file=artifacts/pr-${PR_ID}/test_results_${batch}.json \ - -v $(cat "$batch") || true - done - - # Combine results if needed + if [[ -f "artifacts/pr-${PR_ID}/run_tests.sh" ]]; then + chmod +x artifacts/pr-${PR_ID}/run_tests.sh + bash artifacts/pr-${PR_ID}/run_tests.sh + else + echo "No test script generated." + fi else echo "No remaining tests to run." fi @@ -288,7 +303,10 @@ jobs: uses: actions/upload-artifact@v4 with: name: pr-${{ env.PR_ID }}-test-results - path: artifacts/pr-${{ env.PR_ID }}/test_results.json + path: | + artifacts/pr-${{ env.PR_ID }}/*.json + artifacts/pr-${{ env.PR_ID }}/*.sh + retrieve-results: diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py new file mode 100644 index 00000000000..60a8c5b0c7e --- /dev/null +++ b/scripts/generate_pytest_commands.py @@ -0,0 +1,134 @@ +import json +import os +import sys +import argparse +from pathlib import Path + +def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50): + """ + Create JSON files for test batches that can be used to generate pytest commands. + + Args: + test_list: List of test identifiers + output_dir: Directory to save JSON files + pr_id: PR ID for naming the artifacts + batch_size: Number of tests per batch + """ + # Create output directory if it doesn't exist + output_path = Path(output_dir) / f"pr-{pr_id}" + output_path.mkdir(parents=True, exist_ok=True) + + # Process test identifiers to ensure they're in the correct format + processed_tests = [] + for test in test_list: + # Remove any wrapper if present + if test.strip().startswith(""): + test = test.strip()[10:-1] + processed_tests.append(test.strip()) + + # Split tests into batches + batches = [] + for i in range(0, len(processed_tests), batch_size): + batches.append(processed_tests[i:i+batch_size]) + + # Create JSON files for each batch + batch_files = [] + for i, batch in enumerate(batches): + batch_id = chr(97 + i % 26) * (1 + i // 26) # a, b, c, ... z, aa, bb, etc. + + batch_data = { + "batch_id": batch_id, + "tests": batch, + "command": { + "executable": "pytest", + "options": [ + "--tb=short", + "--json-report", + f"--json-report-file=artifacts/pr-{pr_id}/test_results_batch_{batch_id}.json", + "-v" + ], + "test_identifiers": batch + } + } + + # Save to JSON file + batch_file = output_path / f"batch_{batch_id}.json" + with open(batch_file, 'w') as f: + json.dump(batch_data, f, indent=2) + + batch_files.append(str(batch_file)) + + # Create a manifest file listing all batches + manifest = { + "pr_id": pr_id, + "batch_count": len(batches), + "batch_files": batch_files, + "total_tests": len(processed_tests) + } + + with open(output_path / "manifest.json", 'w') as f: + json.dump(manifest, f, indent=2) + + return str(output_path / "manifest.json") + +def generate_bash_commands(manifest_file, tox_env): + """ + Generate bash commands from the manifest file. + + Args: + manifest_file: Path to the manifest JSON file + tox_env: Tox environment to use + + Returns: + A string containing bash commands + """ + with open(manifest_file, 'r') as f: + manifest = json.load(f) + + commands = [] + commands.append("#!/bin/bash") + commands.append(f"# Test commands for PR-{manifest['pr_id']}") + commands.append(f"# Total batches: {manifest['batch_count']}") + commands.append("") + + for batch_file in manifest['batch_files']: + with open(batch_file, 'r') as f: + batch = json.load(f) + + cmd_parts = ["tox", "-e", tox_env, "--", batch['command']['executable']] + batch['command']['options'] + test_identifiers = " ".join(batch['command']['test_identifiers']) + + commands.append(f"echo 'Running batch {batch['batch_id']}...'") + commands.append(f"{' '.join(cmd_parts)} {test_identifiers} || true") + commands.append("") + + return "\n".join(commands) + +def main(): + parser = argparse.ArgumentParser(description='Generate JSON files for pytest commands') + parser.add_argument('--input', '-i', required=True, help='Input file with test identifiers (one per line)') + parser.add_argument('--output-dir', '-o', default='artifacts', help='Output directory for JSON files') + parser.add_argument('--pr-id', '-p', required=True, help='PR ID for naming artifacts') + parser.add_argument('--batch-size', '-b', type=int, default=50, help='Number of tests per batch') + parser.add_argument('--generate-script', '-g', action='store_true', help='Generate bash script') + parser.add_argument('--prefix', default='', help='Prefix for output files (e.g., "failed" for failed tests)') + parser.add_argument('--tox-env', default='', help='Tox environment to use') + + args = parser.parse_args() + + # Rest of your code... + + # Generate bash script with tox command + if args.generate_script: + bash_commands = generate_bash_commands(manifest_file, args.tox_env) + script_path = Path(args.output_dir) / f"pr-{args.pr_id}" / f"run_{args.prefix}_tests.sh" + + with open(script_path, 'w') as f: + f.write(bash_commands) + + # Make the script executable + os.chmod(script_path, 0o755) + print(f"Created bash script: {script_path}") + +if __name__ == "__main__": + main() From dde7450b2e8839d6b7fe5599a449344283ab70c7 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 17:54:04 -0400 Subject: [PATCH 14/35] fixing issues with scripts --- scripts/generate_pytest_commands.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 60a8c5b0c7e..3976ee8a892 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -4,7 +4,7 @@ import argparse from pathlib import Path -def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50): +def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50, prefix=''): """ Create JSON files for test batches that can be used to generate pytest commands. @@ -63,13 +63,15 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50): "pr_id": pr_id, "batch_count": len(batches), "batch_files": batch_files, - "total_tests": len(processed_tests) + "total_tests": len(processed_tests), + "prefix": prefix } - with open(output_path / "manifest.json", 'w') as f: + manifest_file = output_path / f"{prefix}_manifest.json" if prefix else output_path / "manifest.json" + with open(manifest_file, 'w') as f: json.dump(manifest, f, indent=2) - return str(output_path / "manifest.json") + return str(manifest_file) def generate_bash_commands(manifest_file, tox_env): """ @@ -116,9 +118,22 @@ def main(): args = parser.parse_args() - # Rest of your code... + # Read test identifiers from input file + with open(args.input, 'r') as f: + test_list = [line.strip() for line in f if line.strip()] - # Generate bash script with tox command + # Create JSON files + manifest_file = create_test_batch_json( + test_list, + args.output_dir, + args.pr_id, + args.batch_size, + args.prefix + ) + + print(f"Created manifest file: {manifest_file}") + + # Generate bash script if requested if args.generate_script: bash_commands = generate_bash_commands(manifest_file, args.tox_env) script_path = Path(args.output_dir) / f"pr-{args.pr_id}" / f"run_{args.prefix}_tests.sh" @@ -130,5 +145,6 @@ def main(): os.chmod(script_path, 0o755) print(f"Created bash script: {script_path}") + if __name__ == "__main__": main() From 4adee23663e73c7bc49c3f0cba83c90b19d6a1b2 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 18:01:57 -0400 Subject: [PATCH 15/35] fixing naming issues --- scripts/generate_pytest_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 3976ee8a892..1e97e6da9ec 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -136,7 +136,7 @@ def main(): # Generate bash script if requested if args.generate_script: bash_commands = generate_bash_commands(manifest_file, args.tox_env) - script_path = Path(args.output_dir) / f"pr-{args.pr_id}" / f"run_{args.prefix}_tests.sh" + script_path = Path(args.output_dir) / f"pr-{args.pr_id}" / f"run_tests.sh" with open(script_path, 'w') as f: f.write(bash_commands) From d5962b1b4402d5bc78d588dfc12d2101af24d3c5 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 18:08:30 -0400 Subject: [PATCH 16/35] debugging for run.sh file --- .github/workflows/test.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2845b84bf36..2321d9b0093 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -279,7 +279,18 @@ jobs: shell: bash run: | python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 - + - name: Display Retrieved Test Results + shell: bash + run: | + echo "=======================================" + echo "Retrieved Test Results from PR ${PR_ID}:" + ls -la retrieved-results/ + echo "Test results JSON:" + cat retrieved-results/test_results.json + echo "Shell scripts:" + ls -la retrieved-results/*.sh || echo "No shell scripts found" + echo "=======================================" + - name: Run Remaining Test Cases shell: bash run: | From 3046cfa7881df7011a28794ff02ff9ab20ba9734 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 18:11:38 -0400 Subject: [PATCH 17/35] typos --- .github/workflows/test.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2321d9b0093..f98b65a7e0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -279,18 +279,19 @@ jobs: shell: bash run: | python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 - - name: Display Retrieved Test Results - shell: bash - run: | - echo "=======================================" - echo "Retrieved Test Results from PR ${PR_ID}:" - ls -la retrieved-results/ - echo "Test results JSON:" - cat retrieved-results/test_results.json - echo "Shell scripts:" - ls -la retrieved-results/*.sh || echo "No shell scripts found" - echo "=======================================" - + + - name: Display Retrieved Test Results + shell: bash + run: | + echo "=======================================" + echo "Retrieved Test Results from PR ${PR_ID}:" + ls -la retrieved-results/ + echo "Test results JSON:" + cat retrieved-results/test_results.json + echo "Shell scripts:" + ls -la retrieved-results/*.sh || echo "No shell scripts found" + echo "=======================================" + - name: Run Remaining Test Cases shell: bash run: | From 6353d88d0e7d261ab0c0198222bd4cba5b2102a8 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 18:17:59 -0400 Subject: [PATCH 18/35] debugging --- .github/workflows/test.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f98b65a7e0f..f98604a05f6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -283,14 +283,13 @@ jobs: - name: Display Retrieved Test Results shell: bash run: | - echo "=======================================" - echo "Retrieved Test Results from PR ${PR_ID}:" - ls -la retrieved-results/ - echo "Test results JSON:" - cat retrieved-results/test_results.json - echo "Shell scripts:" - ls -la retrieved-results/*.sh || echo "No shell scripts found" - echo "=======================================" + RUN_TESTS_FILE="artifacts/pr-${PR_ID}/run_tests.sh" + if [[ -f "$RUN_TESTS_FILE" ]]; then + echo "Content of run_tests.sh:" + cat "$RUN_TESTS_FILE" + else + echo "run_tests.sh file does not exist." + fi - name: Run Remaining Test Cases shell: bash From ab3c5a5c9082ad82c28a0608617cb79c12681333 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 18:32:44 -0400 Subject: [PATCH 19/35] improving grep regex --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f98604a05f6..832b7338f2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,7 +166,8 @@ jobs: # Use tox to collect all tests # tox -e $#{{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true - tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | sed -E 's/.*$/\1/' > $ALL_TESTS_FILE || true + tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep -E "<(Function|Class|Module) " | sed -E 's/.*<(Function|Class|Module) ([^>]*)>.*/\2/' > $ALL_TESTS_FILE || true + if [[ -f "$PREV_RESULTS" ]]; then echo "Extracting failed test cases from previous run..." From 88df6a2adfc11d858ad9793b7cb3d46060ff2a94 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 18:50:12 -0400 Subject: [PATCH 20/35] fixing generate test script bugs --- .github/workflows/test.yml | 5 ++--- scripts/generate_pytest_commands.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 832b7338f2e..ee266fa3fa6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,8 +166,7 @@ jobs: # Use tox to collect all tests # tox -e $#{{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true - tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep -E "<(Function|Class|Module) " | sed -E 's/.*<(Function|Class|Module) ([^>]*)>.*/\2/' > $ALL_TESTS_FILE || true - + tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep -E "^(.*?)::" | sed -E 's/\s+.*$//' > $ALL_TESTS_FILE || true if [[ -f "$PREV_RESULTS" ]]; then echo "Extracting failed test cases from previous run..." @@ -212,7 +211,7 @@ jobs: FAILED_TESTS_FILE="artifacts/pr-${PR_ID}/failed_tests.txt" if [[ -s "$FAILED_TESTS_FILE" ]]; then - python scripts/generate_pytest_commands.py --input ${FAILED_TESTS_FILE} --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 --prefix failed + python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 --tox-env ${{ matrix.tox_env }} fi - name: Run Previously Failed Tests First diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 1e97e6da9ec..62af28e098f 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -21,10 +21,17 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50, prefix=' # Process test identifiers to ensure they're in the correct format processed_tests = [] for test in test_list: + # Extract only the test identifier part (remove descriptions) + test = test.strip() + # If it contains a space, take only the part before the space + if ' ' in test: + test = test.split(' ')[0] # Remove any wrapper if present - if test.strip().startswith(""): - test = test.strip()[10:-1] - processed_tests.append(test.strip()) + if test.startswith(""): + test = test[10:-1] + # Only add if it looks like a valid test identifier + if "::" in test or test.endswith(".py"): + processed_tests.append(test) # Split tests into batches batches = [] From a6e10e95e7e31b09ce841c5df51643c9c791f6d8 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 21:41:28 -0400 Subject: [PATCH 21/35] changing grep command --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ee266fa3fa6..60dd2e203d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -165,8 +165,8 @@ jobs: REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt" # Use tox to collect all tests - # tox -e $#{{ matrix.tox_env }} -- --collect-only --quiet | grep "::" > $ALL_TESTS_FILE || true - tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep -E "^(.*?)::" | sed -E 's/\s+.*$//' > $ALL_TESTS_FILE || true + tox -e ${{ matrix.tox_env }} -- --collect-only --quiet | grep -v "SKIP" | grep "::" > $ALL_TESTS_FILE || true + #tox -e $#{{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep -E "^(.*?)::" | sed -E 's/\s+.*$//' > $ALL_TESTS_FILE || true if [[ -f "$PREV_RESULTS" ]]; then echo "Extracting failed test cases from previous run..." From 6b3bafd022738a9fe8561ffa07738788aa3c85cf Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Sun, 6 Apr 2025 21:50:10 -0400 Subject: [PATCH 22/35] fixing bugs --- .github/workflows/test.yml | 4 ++-- scripts/generate_pytest_commands.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60dd2e203d3..ab7e695306f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,7 +166,7 @@ jobs: # Use tox to collect all tests tox -e ${{ matrix.tox_env }} -- --collect-only --quiet | grep -v "SKIP" | grep "::" > $ALL_TESTS_FILE || true - #tox -e $#{{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep -E "^(.*?)::" | sed -E 's/\s+.*$//' > $ALL_TESTS_FILE || true + #tox -e ${{ matrix.tox_env }} -- --collect-only -v | grep -v "SKIP" | grep -E "^(.*?)::" | sed -E 's/\s+.*$//' > $ALL_TESTS_FILE || true if [[ -f "$PREV_RESULTS" ]]; then echo "Extracting failed test cases from previous run..." @@ -278,7 +278,7 @@ jobs: - name: Generate Test Commands shell: bash run: | - python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 + python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 --tox-env ${{ matrix.tox_env }} - name: Display Retrieved Test Results shell: bash diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 62af28e098f..8b2cb757ecf 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -103,8 +103,8 @@ def generate_bash_commands(manifest_file, tox_env): for batch_file in manifest['batch_files']: with open(batch_file, 'r') as f: batch = json.load(f) - - cmd_parts = ["tox", "-e", tox_env, "--", batch['command']['executable']] + batch['command']['options'] + tox_env_param = tox_env if tox_env else "py" + cmd_parts = ["tox", "-e", tox_env_param, "--", batch['command']['executable']] + batch['command']['options'] test_identifiers = " ".join(batch['command']['test_identifiers']) commands.append(f"echo 'Running batch {batch['batch_id']}...'") From 08986978ac2364a83591f4c147a60d7ef508eda8 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 09:21:06 -0400 Subject: [PATCH 23/35] fixing duplicate pytest --- scripts/generate_pytest_commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 8b2cb757ecf..9c46144fd44 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -47,7 +47,6 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50, prefix=' "batch_id": batch_id, "tests": batch, "command": { - "executable": "pytest", "options": [ "--tb=short", "--json-report", From 5959215eaece55e84aaeeb047a4c84e0eac61a12 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 09:25:10 -0400 Subject: [PATCH 24/35] fixing commands --- scripts/generate_pytest_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 9c46144fd44..8a3f189b181 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -47,6 +47,7 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50, prefix=' "batch_id": batch_id, "tests": batch, "command": { + "executable": "pytest", "options": [ "--tb=short", "--json-report", @@ -103,7 +104,7 @@ def generate_bash_commands(manifest_file, tox_env): with open(batch_file, 'r') as f: batch = json.load(f) tox_env_param = tox_env if tox_env else "py" - cmd_parts = ["tox", "-e", tox_env_param, "--", batch['command']['executable']] + batch['command']['options'] + cmd_parts = ["tox", "-e", tox_env_param, "--", ] + batch['command']['options'] test_identifiers = " ".join(batch['command']['test_identifiers']) commands.append(f"echo 'Running batch {batch['batch_id']}...'") From 597e12ec16141cc1bb29c18221bb5f11c767d780 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 09:44:52 -0400 Subject: [PATCH 25/35] excluding soecial characters from test identifiers --- scripts/generate_pytest_commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 8a3f189b181..70301e35e85 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -105,8 +105,7 @@ def generate_bash_commands(manifest_file, tox_env): batch = json.load(f) tox_env_param = tox_env if tox_env else "py" cmd_parts = ["tox", "-e", tox_env_param, "--", ] + batch['command']['options'] - test_identifiers = " ".join(batch['command']['test_identifiers']) - + test_identifiers = " ".join([f"'{t}'" for t in batch['command']['test_identifiers']]) commands.append(f"echo 'Running batch {batch['batch_id']}...'") commands.append(f"{' '.join(cmd_parts)} {test_identifiers} || true") commands.append("") From 25c920a0345ee45a07ed67e7cfbf008ef878da3d Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 10:09:53 -0400 Subject: [PATCH 26/35] fixing grep issues --- scripts/generate_pytest_commands.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 70301e35e85..a4cf5e49a7b 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -91,6 +91,7 @@ def generate_bash_commands(manifest_file, tox_env): Returns: A string containing bash commands """ + with open(manifest_file, 'r') as f: manifest = json.load(f) @@ -103,11 +104,21 @@ def generate_bash_commands(manifest_file, tox_env): for batch_file in manifest['batch_files']: with open(batch_file, 'r') as f: batch = json.load(f) - tox_env_param = tox_env if tox_env else "py" - cmd_parts = ["tox", "-e", tox_env_param, "--", ] + batch['command']['options'] - test_identifiers = " ".join([f"'{t}'" for t in batch['command']['test_identifiers']]) + + cmd_parts = ["tox", "-e", tox_env, "--"] + options = " ".join(batch['command']['options']) + + # Properly escape each test identifier + test_lines = [] + for test in batch['command']['test_identifiers']: + # Double quote each test identifier and escape any internal quotes + escaped_test = test.replace("'", "'\\''") + test_lines.append(f" '{escaped_test}'") + + + test_str = " \\\n".join(test_lines) commands.append(f"echo 'Running batch {batch['batch_id']}...'") - commands.append(f"{' '.join(cmd_parts)} {test_identifiers} || true") + commands.append(f"{' '.join(cmd_parts)} {options} {test_str} || true") commands.append("") return "\n".join(commands) From 3490392471c69bcf58acef415f89287a6bfee7ac Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 10:31:33 -0400 Subject: [PATCH 27/35] reducing batch size --- scripts/generate_pytest_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index a4cf5e49a7b..ca12d4c0da3 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -4,7 +4,7 @@ import argparse from pathlib import Path -def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50, prefix=''): +def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=''): """ Create JSON files for test batches that can be used to generate pytest commands. @@ -41,7 +41,7 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=50, prefix=' # Create JSON files for each batch batch_files = [] for i, batch in enumerate(batches): - batch_id = chr(97 + i % 26) * (1 + i // 26) # a, b, c, ... z, aa, bb, etc. + batch_id = str(i + 1) # 1, 2, 3, 4, etc. batch_data = { "batch_id": batch_id, From 9fbf6e6cbf54daffca03c625a7dc84c769ea5421 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 18:02:47 -0400 Subject: [PATCH 28/35] fixing command line length issues --- .github/workflows/test.yml | 9 +++-- scripts/generate_pytest_commands.py | 60 ++++++++--------------------- tox.ini | 5 +++ 3 files changed, 28 insertions(+), 46 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab7e695306f..4d932fa98d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -295,13 +295,16 @@ jobs: shell: bash run: | REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt" - if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." - if [[ -f "artifacts/pr-${PR_ID}/run_tests.sh" ]]; then chmod +x artifacts/pr-${PR_ID}/run_tests.sh - bash artifacts/pr-${PR_ID}/run_tests.sh + # Split the run_tests.sh into smaller chunks if it's too large + split -l 100 artifacts/pr-${PR_ID}/run_tests.sh artifacts/pr-${PR_ID}/run_tests_chunk_ + for chunk in artifacts/pr-${PR_ID}/run_tests_chunk_*; do + echo "Running chunk $chunk" + bash "$chunk" + done else echo "No test script generated." fi diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index ca12d4c0da3..90b4ad03995 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -18,52 +18,26 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=' output_path = Path(output_dir) / f"pr-{pr_id}" output_path.mkdir(parents=True, exist_ok=True) - # Process test identifiers to ensure they're in the correct format - processed_tests = [] - for test in test_list: - # Extract only the test identifier part (remove descriptions) - test = test.strip() - # If it contains a space, take only the part before the space - if ' ' in test: - test = test.split(' ')[0] - # Remove any wrapper if present - if test.startswith(""): - test = test[10:-1] - # Only add if it looks like a valid test identifier - if "::" in test or test.endswith(".py"): - processed_tests.append(test) + test_modules = {} + for test in processed_tests: + module = test.split("::")[0] + if module not in test_modules: + test_modules[module] = [] + test_modules[module].append(test) # Split tests into batches batches = [] - for i in range(0, len(processed_tests), batch_size): - batches.append(processed_tests[i:i+batch_size]) - - # Create JSON files for each batch - batch_files = [] - for i, batch in enumerate(batches): - batch_id = str(i + 1) # 1, 2, 3, 4, etc. - - batch_data = { - "batch_id": batch_id, - "tests": batch, - "command": { - "executable": "pytest", - "options": [ - "--tb=short", - "--json-report", - f"--json-report-file=artifacts/pr-{pr_id}/test_results_batch_{batch_id}.json", - "-v" - ], - "test_identifiers": batch - } - } - - # Save to JSON file - batch_file = output_path / f"batch_{batch_id}.json" - with open(batch_file, 'w') as f: - json.dump(batch_data, f, indent=2) - - batch_files.append(str(batch_file)) + current_batch = [] + current_size = 0 + for module, tests in test_modules.items(): + if current_size + len(tests) > batch_size and current_batch: + batches.append(current_batch) + current_batch = [] + current_size = 0 + current_batch.extend(tests) + current_size += len(tests) + if current_batch: + batches.append(current_batch) # Create a manifest file listing all batches manifest = { diff --git a/tox.ini b/tox.ini index 1a28816b45f..354ee40faa6 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,12 @@ passenv = SETUPTOOLS_SCM_PRETEND_VERSION setenv = _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} + PYTEST_ADDOPTS = --tb=short PYTHONWARNDEFAULTENCODING=1 + PYTEST_ADDOPTS = --tb=short + # Increase command line length limit for Windows + PYTHONUTF8 = 1 + PYTHONIOENCODING = utf-8 # Coverage configuration coverage: _PYTEST_TOX_COVERAGE_RUN=coverage run -m coverage: _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess From 6072066eb91ca4b3e0893ae1dbed52a9cacdab46 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 18:09:07 -0400 Subject: [PATCH 29/35] fixing typos --- scripts/generate_pytest_commands.py | 82 +++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 17 deletions(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 90b4ad03995..8e91ece09b6 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -13,22 +13,40 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=' output_dir: Directory to save JSON files pr_id: PR ID for naming the artifacts batch_size: Number of tests per batch + prefix: Prefix for output files """ # Create output directory if it doesn't exist output_path = Path(output_dir) / f"pr-{pr_id}" output_path.mkdir(parents=True, exist_ok=True) + # Process test identifiers to ensure they're in the correct format + processed_tests = [] + for test in test_list: + # Extract only the test identifier part (remove descriptions) + test = test.strip() + # If it contains a space, take only the part before the space + if ' ' in test: + test = test.split(' ')[0] + # Remove any wrapper if present + if test.startswith(""): + test = test[10:-1] + # Only add if it looks like a valid test identifier + if "::" in test or test.endswith(".py"): + processed_tests.append(test) + + # Group tests by module to reduce command line complexity test_modules = {} for test in processed_tests: - module = test.split("::")[0] + module = test.split("::")[0] if "::" in test else test if module not in test_modules: test_modules[module] = [] test_modules[module].append(test) - # Split tests into batches + # Create batches based on modules to avoid command line length issues batches = [] current_batch = [] current_size = 0 + for module, tests in test_modules.items(): if current_size + len(tests) > batch_size and current_batch: batches.append(current_batch) @@ -36,9 +54,37 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=' current_size = 0 current_batch.extend(tests) current_size += len(tests) + if current_batch: batches.append(current_batch) + # Create JSON files for each batch + batch_files = [] + for i, batch in enumerate(batches): + batch_id = str(i + 1) # 1, 2, 3, 4, etc. + + batch_data = { + "batch_id": batch_id, + "tests": batch, + "command": { + "executable": "pytest", + "options": [ + "--tb=short", + "--json-report", + f"--json-report-file=artifacts/pr-{pr_id}/test_results_batch_{batch_id}.json", + "-v" + ], + "test_identifiers": batch + } + } + + # Save to JSON file + batch_file = output_path / f"batch_{batch_id}.json" + with open(batch_file, 'w') as f: + json.dump(batch_data, f, indent=2) + + batch_files.append(str(batch_file)) + # Create a manifest file listing all batches manifest = { "pr_id": pr_id, @@ -65,7 +111,6 @@ def generate_bash_commands(manifest_file, tox_env): Returns: A string containing bash commands """ - with open(manifest_file, 'r') as f: manifest = json.load(f) @@ -79,20 +124,24 @@ def generate_bash_commands(manifest_file, tox_env): with open(batch_file, 'r') as f: batch = json.load(f) - cmd_parts = ["tox", "-e", tox_env, "--"] - options = " ".join(batch['command']['options']) + # Create command with proper line breaks for readability + commands.append(f"echo 'Running batch {batch['batch_id']}...'") + commands.append(f"timeout 60s tox -e {tox_env} -- \\") + commands.append(" --tb=short \\") + commands.append(" --json-report \\") + commands.append(f" --json-report-file=artifacts/pr-{manifest['pr_id']}/test_results_batch_{batch['batch_id']}.json \\") + commands.append(" -v \\") - # Properly escape each test identifier + # Add test identifiers with proper escaping test_lines = [] for test in batch['command']['test_identifiers']: - # Double quote each test identifier and escape any internal quotes + # Escape any special characters in test names escaped_test = test.replace("'", "'\\''") test_lines.append(f" '{escaped_test}'") - - test_str = " \\\n".join(test_lines) - commands.append(f"echo 'Running batch {batch['batch_id']}...'") - commands.append(f"{' '.join(cmd_parts)} {options} {test_str} || true") + # Join all test identifiers with line continuation + test_str = " \\\n".join(test_lines) + commands.append(test_str + " || true") commands.append("") return "\n".join(commands) @@ -102,7 +151,7 @@ def main(): parser.add_argument('--input', '-i', required=True, help='Input file with test identifiers (one per line)') parser.add_argument('--output-dir', '-o', default='artifacts', help='Output directory for JSON files') parser.add_argument('--pr-id', '-p', required=True, help='PR ID for naming artifacts') - parser.add_argument('--batch-size', '-b', type=int, default=50, help='Number of tests per batch') + parser.add_argument('--batch-size', '-b', type=int, default=20, help='Number of tests per batch') parser.add_argument('--generate-script', '-g', action='store_true', help='Generate bash script') parser.add_argument('--prefix', default='', help='Prefix for output files (e.g., "failed" for failed tests)') parser.add_argument('--tox-env', default='', help='Tox environment to use') @@ -115,9 +164,9 @@ def main(): # Create JSON files manifest_file = create_test_batch_json( - test_list, - args.output_dir, - args.pr_id, + test_list, + args.output_dir, + args.pr_id, args.batch_size, args.prefix ) @@ -127,7 +176,7 @@ def main(): # Generate bash script if requested if args.generate_script: bash_commands = generate_bash_commands(manifest_file, args.tox_env) - script_path = Path(args.output_dir) / f"pr-{args.pr_id}" / f"run_tests.sh" + script_path = Path(args.output_dir) / f"pr-{args.pr_id}" / f"run_{args.prefix}_tests.sh" if args.prefix else Path(args.output_dir) / f"pr-{args.pr_id}" / "run_tests.sh" with open(script_path, 'w') as f: f.write(bash_commands) @@ -136,6 +185,5 @@ def main(): os.chmod(script_path, 0o755) print(f"Created bash script: {script_path}") - if __name__ == "__main__": main() From b0e832c4a4328193e5f199ae2ae88acd775b3da1 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 18:19:31 -0400 Subject: [PATCH 30/35] change to extract test identifiers from the function wrapper format that pytest uses in its collection output. From b286ec104c74b0f9b857e2b56655bbf0335b24b1 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 18:30:36 -0400 Subject: [PATCH 31/35] reverting changes --- scripts/generate_pytest_commands.py | 34 ++++++----------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 8e91ece09b6..078cc0cd09c 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -18,7 +18,7 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=' # Create output directory if it doesn't exist output_path = Path(output_dir) / f"pr-{pr_id}" output_path.mkdir(parents=True, exist_ok=True) - + # Process test identifiers to ensure they're in the correct format processed_tests = [] for test in test_list: @@ -27,36 +27,17 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=' # If it contains a space, take only the part before the space if ' ' in test: test = test.split(' ')[0] - # Remove any wrapper if present + # Remove any wrapper if present if test.startswith(""): test = test[10:-1] # Only add if it looks like a valid test identifier if "::" in test or test.endswith(".py"): processed_tests.append(test) - - # Group tests by module to reduce command line complexity - test_modules = {} - for test in processed_tests: - module = test.split("::")[0] if "::" in test else test - if module not in test_modules: - test_modules[module] = [] - test_modules[module].append(test) - - # Create batches based on modules to avoid command line length issues + + # Split tests into batches batches = [] - current_batch = [] - current_size = 0 - - for module, tests in test_modules.items(): - if current_size + len(tests) > batch_size and current_batch: - batches.append(current_batch) - current_batch = [] - current_size = 0 - current_batch.extend(tests) - current_size += len(tests) - - if current_batch: - batches.append(current_batch) + for i in range(0, len(processed_tests), batch_size): + batches.append(processed_tests[i:i+batch_size]) # Create JSON files for each batch batch_files = [] @@ -83,8 +64,7 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=' with open(batch_file, 'w') as f: json.dump(batch_data, f, indent=2) - batch_files.append(str(batch_file)) - + batch_files.append(str(batch_file)) # Create a manifest file listing all batches manifest = { "pr_id": pr_id, From 442bea11238edede6d6977fd0347b2fb4e53f1d9 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 18:51:51 -0400 Subject: [PATCH 32/35] reverting changes --- .github/workflows/test.yml | 9 +++------ tox.ini | 1 - 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d932fa98d3..ab7e695306f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -295,16 +295,13 @@ jobs: shell: bash run: | REMAINING_TESTS_FILE="artifacts/pr-${PR_ID}/remaining_tests.txt" + if [[ -s "$REMAINING_TESTS_FILE" ]]; then echo "Running remaining test cases using tox env ${{ matrix.tox_env }}..." + if [[ -f "artifacts/pr-${PR_ID}/run_tests.sh" ]]; then chmod +x artifacts/pr-${PR_ID}/run_tests.sh - # Split the run_tests.sh into smaller chunks if it's too large - split -l 100 artifacts/pr-${PR_ID}/run_tests.sh artifacts/pr-${PR_ID}/run_tests_chunk_ - for chunk in artifacts/pr-${PR_ID}/run_tests_chunk_*; do - echo "Running chunk $chunk" - bash "$chunk" - done + bash artifacts/pr-${PR_ID}/run_tests.sh else echo "No test script generated." fi diff --git a/tox.ini b/tox.ini index 354ee40faa6..c090602d77d 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,6 @@ passenv = SETUPTOOLS_SCM_PRETEND_VERSION setenv = _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} - PYTEST_ADDOPTS = --tb=short PYTHONWARNDEFAULTENCODING=1 PYTEST_ADDOPTS = --tb=short # Increase command line length limit for Windows From 54bb33fc732381f451afa44384c3cab18ac18300 Mon Sep 17 00:00:00 2001 From: shonilbhide Date: Mon, 7 Apr 2025 20:43:22 -0400 Subject: [PATCH 33/35] Fix test identifier processing and add workflow ID support to prevent artifact conflicts --- .github/workflows/test.yml | 13 ++-- scripts/generate_pytest_commands.py | 116 ++++++++++++++++++++++++++-- 2 files changed, 117 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab7e695306f..76dc2f8b2b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -275,10 +275,14 @@ jobs: echo "No remaining tests to run." fi + - name: Set Workflow ID + shell: bash + run: echo "WORKFLOW_ID=${{ matrix.name }}" >> $GITHUB_ENV + - name: Generate Test Commands shell: bash run: | - python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --generate-script --batch-size 50 --tox-env ${{ matrix.tox_env }} + python scripts/generate_pytest_commands.py --input artifacts/pr-${PR_ID}/remaining_tests.txt --output-dir artifacts --pr-id ${PR_ID} --workflow-id ${WORKFLOW_ID} --generate-script --batch-size 20 --tox-env ${{ matrix.tox_env }} - name: Display Retrieved Test Results shell: bash @@ -309,14 +313,13 @@ jobs: echo "No remaining tests to run." fi - - name: Upload New Test Results uses: actions/upload-artifact@v4 with: - name: pr-${{ env.PR_ID }}-test-results + name: pr-${{ env.PR_ID }}-${{ env.WORKFLOW_ID }}-test-results path: | - artifacts/pr-${{ env.PR_ID }}/*.json - artifacts/pr-${{ env.PR_ID }}/*.sh + artifacts/pr-${{ env.PR_ID }}/${{ env.WORKFLOW_ID }}/*.json + artifacts/pr-${{ env.PR_ID }}/${{ env.WORKFLOW_ID }}/*.sh diff --git a/scripts/generate_pytest_commands.py b/scripts/generate_pytest_commands.py index 078cc0cd09c..92f509d43ea 100644 --- a/scripts/generate_pytest_commands.py +++ b/scripts/generate_pytest_commands.py @@ -1,10 +1,98 @@ import json import os import sys +import glob import argparse from pathlib import Path -def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=''): +def combine_test_results(pr_id, workflow_id, output_dir="artifacts"): + """ + Combine all batch test results into a single JSON file. + + Args: + pr_id: PR ID for naming the artifacts + workflow_id: Unique ID for the workflow in the matrix + output_dir: Directory containing the artifacts + """ + output_path = Path(output_dir) / f"pr-{pr_id}" / workflow_id + + # Find all batch result files + batch_files = list(output_path.glob("test_results_batch_*.json")) + + if not batch_files: + print(f"No batch result files found in {output_path}") + return + + # Initialize combined results + combined_results = { + "created": None, + "duration": 0, + "exitcode": 0, + "root": None, + "environment": {}, + "summary": { + "passed": 0, + "failed": 0, + "skipped": 0, + "xfailed": 0, + "xpassed": 0, + "error": 0, + "total": 0 + }, + "tests": [], + "collectors": [], + "warnings": [] + } + + # Process each batch file + for batch_file in batch_files: + try: + with open(batch_file, 'r') as f: + batch_data = json.load(f) + + # Update summary + for key in combined_results["summary"]: + if key in batch_data["summary"]: + combined_results["summary"][key] += batch_data["summary"][key] + + # Add tests + combined_results["tests"].extend(batch_data.get("tests", [])) + + # Add collectors + combined_results["collectors"].extend(batch_data.get("collectors", [])) + + # Add warnings + combined_results["warnings"].extend(batch_data.get("warnings", [])) + + # Update duration + combined_results["duration"] += batch_data.get("duration", 0) + + # Update exitcode (non-zero takes precedence) + if batch_data.get("exitcode", 0) != 0: + combined_results["exitcode"] = batch_data["exitcode"] + + # Use the first batch's created timestamp and root + if combined_results["created"] is None and "created" in batch_data: + combined_results["created"] = batch_data["created"] + + if combined_results["root"] is None and "root" in batch_data: + combined_results["root"] = batch_data["root"] + + # Merge environment info + combined_results["environment"].update(batch_data.get("environment", {})) + + except Exception as e: + print(f"Error processing {batch_file}: {e}") + + # Save combined results + combined_file = output_path / "test_results.json" + with open(combined_file, 'w') as f: + json.dump(combined_results, f, indent=2) + + print(f"Combined {len(batch_files)} batch results into {combined_file}") + + +def create_test_batch_json(test_list, output_dir, pr_id, workflow_id, batch_size=20, prefix=''): """ Create JSON files for test batches that can be used to generate pytest commands. @@ -16,7 +104,7 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=' prefix: Prefix for output files """ # Create output directory if it doesn't exist - output_path = Path(output_dir) / f"pr-{pr_id}" + output_path = Path(output_dir) / f"pr-{pr_id}" / workflow_id output_path.mkdir(parents=True, exist_ok=True) # Process test identifiers to ensure they're in the correct format @@ -52,7 +140,7 @@ def create_test_batch_json(test_list, output_dir, pr_id, batch_size=20, prefix=' "options": [ "--tb=short", "--json-report", - f"--json-report-file=artifacts/pr-{pr_id}/test_results_batch_{batch_id}.json", + f"--json-report-file=artifacts/pr-{pr_id}/{workflow_id}/test_results_batch_{batch_id}.json", "-v" ], "test_identifiers": batch @@ -87,7 +175,7 @@ def generate_bash_commands(manifest_file, tox_env): Args: manifest_file: Path to the manifest JSON file tox_env: Tox environment to use - + Returns: A string containing bash commands """ @@ -124,6 +212,11 @@ def generate_bash_commands(manifest_file, tox_env): commands.append(test_str + " || true") commands.append("") + # Add command to combine all batch results into a single file + commands.append("# Combine all batch results into a single file") + commands.append(f"python {os.path.abspath(__file__)} --combine-results --output-dir=artifacts --pr-id={manifest['pr_id']}") + commands.append("") + return "\n".join(commands) def main(): @@ -135,8 +228,17 @@ def main(): parser.add_argument('--generate-script', '-g', action='store_true', help='Generate bash script') parser.add_argument('--prefix', default='', help='Prefix for output files (e.g., "failed" for failed tests)') parser.add_argument('--tox-env', default='', help='Tox environment to use') - + parser.add_argument('--workflow-id', '-w', required=True, help='Unique ID for the workflow in the matrix') + parser.add_argument('--combine-results', action='store_true', help='Combine batch results into a single file') + args = parser.parse_args() + + if args.combine_results: + combine_test_results(args.pr_id, args.workflow_id, args.output_dir) + return + + if not args.input: + parser.error("--input is required unless --combine-results is specified") # Read test identifiers from input file with open(args.input, 'r') as f: @@ -147,6 +249,7 @@ def main(): test_list, args.output_dir, args.pr_id, + args.workflow_id, args.batch_size, args.prefix ) @@ -156,8 +259,7 @@ def main(): # Generate bash script if requested if args.generate_script: bash_commands = generate_bash_commands(manifest_file, args.tox_env) - script_path = Path(args.output_dir) / f"pr-{args.pr_id}" / f"run_{args.prefix}_tests.sh" if args.prefix else Path(args.output_dir) / f"pr-{args.pr_id}" / "run_tests.sh" - + script_path = Path(args.output_dir) / f"pr-{args.pr_id}" / args.workflow_id / f"run_{args.prefix}_tests.sh" if args.prefix else Path(args.output_dir) / f"pr-{args.pr_id}" / args.workflow_id / "run_tests.sh" with open(script_path, 'w') as f: f.write(bash_commands) From 9d07d496875cf9e5e17148c4b65cdf886de874ab Mon Sep 17 00:00:00 2001 From: ShubhamDesai <42180509+ShubhamDesai@users.noreply.github.com> Date: Sat, 12 Apr 2025 22:00:31 -0400 Subject: [PATCH 34/35] Fix naming --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76dc2f8b2b7..6d3010568fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -338,7 +338,7 @@ jobs: - name: Download Test Results uses: actions/download-artifact@v4 with: - name: pr-${{ env.PR_ID }}-test-results + name: pr-${{ env.PR_ID }}-${{ env.WORKFLOW_ID }}-test-results path: retrieved-results - name: Display Retrieved Test Results From cdf272c4bc038286a996789c86bdffc8e44a9e91 Mon Sep 17 00:00:00 2001 From: ShubhamDesai <42180509+ShubhamDesai@users.noreply.github.com> Date: Sat, 12 Apr 2025 22:04:37 -0400 Subject: [PATCH 35/35] adding dev branch --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d3010568fd..611e76f8007 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - dev - "[0-9]+.[0-9]+.x" - "test-me-*" tags: @@ -13,6 +14,7 @@ on: pull_request: branches: - main + - dev - "[0-9]+.[0-9]+.x" types: [opened, synchronize, reopened, ready_for_review]