diff --git a/.github/scripts/ci-failfast-watcher.sh b/.github/scripts/ci-failfast-watcher.sh index 192035d5..d05921fe 100755 --- a/.github/scripts/ci-failfast-watcher.sh +++ b/.github/scripts/ci-failfast-watcher.sh @@ -2,8 +2,11 @@ # .github/scripts/ci-failfast-watcher.sh # # Background watcher for self-hosted runners: polls GitHub API every 15s to -# check whether any sibling job in the same workflow run has failed. If so, -# kills the foreground process group and exits 1. +# check whether any sibling job in the same workflow run has a REAL test/lint +# failure. If so, kills the foreground process group and exits 1. +# +# Infrastructure failures (checkout, install, setup) are ignored — those +# should be retried by ci-auto-retry.yaml, not used to cancel healthy jobs. # # Usage: ci-failfast-watcher.sh [args...] # @@ -12,11 +15,24 @@ set -euo pipefail check_sibling_failed() { python3 - <<'PY' -import json, os, sys, urllib.request +import json, os, re, sys, urllib.request + token = os.environ["GH_TOKEN"] repo = os.environ["GITHUB_REPOSITORY"] run_id = os.environ["GITHUB_RUN_ID"] -bad = {"failure", "cancelled", "timed_out", "startup_failure"} + +# Conclusions that indicate a job finished badly +bad = {"failure", "timed_out"} + +# Step name patterns for REAL test/lint failures. +# Only these warrant killing sibling jobs. +TEST_STEP_RE = re.compile( + r"Run .* tests|Run .* coverage|Run .* suite|" + r"Lint with pylint|Assert NPU|Verify MPS|" + r"Run .*card HCCL", + re.IGNORECASE, +) + req = urllib.request.Request( f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100", headers={ @@ -30,10 +46,28 @@ try: data = json.load(resp) except Exception: sys.exit(1) # network error → don't kill + for job in data.get("jobs", []): - if job.get("conclusion") in bad: - print(f"[failfast] sibling failed: {job.get('name')}", file=sys.stderr) + if job.get("conclusion") not in bad: + continue + # Check which steps actually failed + failed_steps = [ + s.get("name", "") + for s in job.get("steps", []) + if s.get("conclusion") == "failure" + ] + # Job failed with no completed steps (runner allocation) → infra, skip + if not failed_steps: + continue + # Only kill if a real test/lint step failed + if any(TEST_STEP_RE.search(name) for name in failed_steps): + print(f"[failfast] sibling real failure: {job.get('name')} " + f"(steps: {failed_steps})", file=sys.stderr) sys.exit(0) + # Infra failure (checkout, install, etc.) → ignore + print(f"[failfast] sibling infra failure (ignored): {job.get('name')} " + f"(steps: {failed_steps})", file=sys.stderr) + sys.exit(1) PY } diff --git a/.github/workflows/ci-auto-retry.yaml b/.github/workflows/ci-auto-retry.yaml index 05189d23..e1964623 100644 --- a/.github/workflows/ci-auto-retry.yaml +++ b/.github/workflows/ci-auto-retry.yaml @@ -55,27 +55,32 @@ jobs: # Real test/lint steps — if failure is here, it's a genuine failure # Pattern covers all test execution and validation steps in ci.yaml - TEST_PATTERN="Run .* tests|Run .* coverage|Run .* suite|Lint with pylint|Assert NPU|Verify MPS" + TEST_PATTERN="Run .* tests|Run .* coverage|Run .* suite|Lint with pylint|Assert NPU|Verify MPS|Run .*card HCCL" + # Classify each failed job as "test" or "infra" HAS_TEST_FAILURE=$(echo "$FAILED_JOBS" | jq --arg pat "$TEST_PATTERN" \ - '[.[].failed_steps[] | test($pat; "i")] | any') + '[.[] | select([.failed_steps[] | test($pat; "i")] | any)] | length > 0') + HAS_INFRA_FAILURE=$(echo "$FAILED_JOBS" | jq --arg pat "$TEST_PATTERN" \ + '[.[] | select((.failed_steps | length == 0) or ([.failed_steps[] | test($pat; "i")] | any | not))] | length > 0') echo "::endgroup::" - if [ "$HAS_TEST_FAILURE" = "true" ]; then - echo "::notice::Real test/lint failure detected — not retrying." + if [ "$HAS_INFRA_FAILURE" != "true" ]; then + echo "::notice::All failures are real test/lint failures — not retrying." exit 0 fi - # All failures are infrastructure-related (setup, checkout, install, OOM, network) - echo "::notice::Only infrastructure failures detected — retrying (attempt $ATTEMPT → $((ATTEMPT + 1)))" + # There are infrastructure failures — retry. + # gh run rerun --failed will re-run all failed jobs; real test failures + # will fail again but infra failures get another chance. + echo "::notice::Infrastructure failures detected — retrying (attempt $ATTEMPT → $((ATTEMPT + 1)))" if [ "$HAS_CANCELLED" = "true" ]; then - # Sibling jobs were cancelled by cancel-on-failure step — must re-run entire workflow + # Sibling jobs were cancelled — must re-run entire workflow echo "Cancelled sibling jobs detected — re-running entire workflow." gh run rerun "$RUN_ID" --repo "$REPO" else - # Only specific jobs failed, others completed — re-run just the failed ones + # Re-run only failed jobs echo "Re-running only failed jobs." gh run rerun "$RUN_ID" --failed --repo "$REPO" fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6f608791..5a474152 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -210,10 +210,11 @@ jobs: if: always() run: ccache -s - name: Run CPU tests + id: run-tests run: | pytest tests/cpu/ tests/contract/ -v --tb=short - name: Cancel workflow on failure - if: failure() + if: steps.run-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: @@ -291,10 +292,11 @@ jobs: run: | python -c "from candle._backends.mps.runtime import is_available; print('MPS available:', is_available())" - name: Run MPS tests + id: run-tests run: | pytest tests/mps/ -v --tb=short - name: Cancel workflow on failure - if: failure() + if: steps.run-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: @@ -348,13 +350,14 @@ jobs: if: always() run: ccache -s - name: Run NPU common + 910B op tests + id: run-tests run: | source /opt/miniconda3/etc/profile.d/conda.sh conda activate candle bash .github/scripts/ci-failfast-watcher.sh \ pytest tests/npu/common/ tests/npu/910b/ -v --tb=short - name: Cancel workflow on failure - if: failure() + if: steps.run-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: @@ -406,6 +409,7 @@ jobs: if: always() run: ccache -s - name: Run NPU integration tests + id: run-tests run: | source /opt/miniconda3/etc/profile.d/conda.sh conda activate candle @@ -417,7 +421,7 @@ jobs: --ignore=tests/npu/310b \ --ignore=tests/npu/310p - name: Cancel workflow on failure - if: failure() + if: steps.run-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: @@ -470,13 +474,14 @@ jobs: if: always() run: ccache -s - name: Run HCCL distributed tests + id: run-tests run: | source /opt/miniconda3/etc/profile.d/conda.sh conda activate candle bash .github/scripts/ci-failfast-watcher.sh \ pytest tests/distributed/ -v --tb=short -k "hccl" - name: Cancel workflow on failure - if: failure() + if: steps.run-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: @@ -569,6 +574,7 @@ jobs: } clean_env python -c "import candle as torch; count = torch.npu.device_count(); assert torch.npu.is_available(), 'expected NPU to be available in NPU Suite (6-7)'; assert count >= 2, f'expected >=2 visible NPUs, found {count}'" - name: Run NPU tests + id: run-tests run: | source "$ASCEND_ENV_SCRIPT" clean_env() { @@ -602,7 +608,7 @@ jobs: --ignore=tests/npu/310b/ \ --ignore=tests/npu/310p/ - name: Cancel workflow on failure - if: failure() + if: steps.run-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: @@ -693,6 +699,7 @@ jobs: } clean_env python -c "import candle as torch; count = torch.npu.device_count(); assert torch.npu.is_available(), 'expected NPU to be available in Distributed Suite (4-5)'; assert count >= 2, f'expected >=2 visible NPUs, found {count}'" - name: Run distributed baseline suite + id: run-dist-tests run: | source "$ASCEND_ENV_SCRIPT" clean_env() { @@ -722,6 +729,7 @@ jobs: clean_env bash .github/scripts/ci-failfast-watcher.sh \ pytest tests/distributed/ -v --tb=short -k "not all_to_all_single_async_unequal_multicard and not all_to_all_single_invalid_split_pairing_multicard and not all_to_all_single_split_numel_validation_multicard and not test_ddp" - name: Run 2-card HCCL all_to_all coverage + id: run-hccl-tests run: | set -o pipefail source "$ASCEND_ENV_SCRIPT" @@ -760,7 +768,7 @@ jobs: exit 1 fi - name: Cancel workflow on failure - if: failure() + if: steps.run-dist-tests.outcome == 'failure' || steps.run-hccl-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: @@ -851,6 +859,7 @@ jobs: } clean_env python -c "import candle as torch; count = torch.npu.device_count(); assert torch.npu.is_available(), 'expected NPU to be available in HCCL 4-Card Suite (0-3)'; assert count >= 4, f'expected >=4 visible NPUs, found {count}'" - name: Run 4-card HCCL all_to_all coverage + id: run-hccl-4card-tests run: | set -o pipefail source "$ASCEND_ENV_SCRIPT" @@ -889,7 +898,7 @@ jobs: exit 1 fi - name: Cancel workflow on failure - if: failure() + if: steps.run-hccl-4card-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: @@ -1011,6 +1020,7 @@ jobs: } clean_env python -c "import candle as torch; assert torch.npu.is_available(), 'NPU not available on 310B runner'; print('NPU available:', torch.npu.is_available()); print('Device count:', torch.npu.device_count())" - name: Run 310B-specific tests + id: run-310b-tests run: | source "$ASCEND_ENV_SCRIPT" clean_env() { @@ -1043,6 +1053,7 @@ jobs: sleep 2 sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null || true - name: Run NPU common tests + id: run-common-tests run: | source "$ASCEND_ENV_SCRIPT" clean_env() { @@ -1070,7 +1081,7 @@ jobs: clean_env bash .github/scripts/ci-failfast-watcher.sh \ pytest tests/npu/common/ -v --tb=short - name: Cancel workflow on failure - if: failure() + if: steps.run-310b-tests.outcome == 'failure' || steps.run-common-tests.outcome == 'failure' run: | gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true env: