fix(ci): don't cancel healthy jobs on sibling infra failures

.github/scripts/ci-failfast-watcher.sh

@@ -2,8 +2,11 @@
 # .github/scripts/ci-failfast-watcher.sh
 #
 # Background watcher for self-hosted runners: polls GitHub API every 15s to
-# check whether any sibling job in the same workflow run has failed.  If so,
-# kills the foreground process group and exits 1.
+# check whether any sibling job in the same workflow run has a REAL test/lint
+# failure.  If so, kills the foreground process group and exits 1.
+#
+# Infrastructure failures (checkout, install, setup) are ignored — those
+# should be retried by ci-auto-retry.yaml, not used to cancel healthy jobs.
 #
 # Usage:  ci-failfast-watcher.sh <command> [args...]
 #
@@ -12,11 +15,24 @@ set -euo pipefail
 
 check_sibling_failed() {
   python3 - <<'PY'
-import json, os, sys, urllib.request
+import json, os, re, sys, urllib.request
+
 token = os.environ["GH_TOKEN"]
 repo  = os.environ["GITHUB_REPOSITORY"]
 run_id = os.environ["GITHUB_RUN_ID"]
-bad = {"failure", "cancelled", "timed_out", "startup_failure"}
+
+# Conclusions that indicate a job finished badly
+bad = {"failure", "timed_out"}
+
+# Step name patterns for REAL test/lint failures.
+# Only these warrant killing sibling jobs.
+TEST_STEP_RE = re.compile(
+    r"Run .* tests|Run .* coverage|Run .* suite|"
+    r"Lint with pylint|Assert NPU|Verify MPS|"
+    r"Run .*card HCCL",
+    re.IGNORECASE,
+)
+
 req = urllib.request.Request(
     f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100",
     headers={
@@ -30,10 +46,28 @@ try:
         data = json.load(resp)
 except Exception:
     sys.exit(1)  # network error → don't kill
+
 for job in data.get("jobs", []):
-    if job.get("conclusion") in bad:
-        print(f"[failfast] sibling failed: {job.get('name')}", file=sys.stderr)
+    if job.get("conclusion") not in bad:
+        continue
+    # Check which steps actually failed
+    failed_steps = [
+        s.get("name", "")
+        for s in job.get("steps", [])
+        if s.get("conclusion") == "failure"
+    ]
+    # Job failed with no completed steps (runner allocation) → infra, skip
+    if not failed_steps:
+        continue
+    # Only kill if a real test/lint step failed
+    if any(TEST_STEP_RE.search(name) for name in failed_steps):
+        print(f"[failfast] sibling real failure: {job.get('name')} "
+              f"(steps: {failed_steps})", file=sys.stderr)
         sys.exit(0)
+    # Infra failure (checkout, install, etc.) → ignore
+    print(f"[failfast] sibling infra failure (ignored): {job.get('name')} "
+          f"(steps: {failed_steps})", file=sys.stderr)
+
 sys.exit(1)
 PY
 }

.github/workflows/ci-auto-retry.yaml

@@ -55,27 +55,32 @@ jobs:
 
           # Real test/lint steps — if failure is here, it's a genuine failure
           # Pattern covers all test execution and validation steps in ci.yaml
-          TEST_PATTERN="Run .* tests|Run .* coverage|Run .* suite|Lint with pylint|Assert NPU|Verify MPS"
+          TEST_PATTERN="Run .* tests|Run .* coverage|Run .* suite|Lint with pylint|Assert NPU|Verify MPS|Run .*card HCCL"
 
+          # Classify each failed job as "test" or "infra"
           HAS_TEST_FAILURE=$(echo "$FAILED_JOBS" | jq --arg pat "$TEST_PATTERN" \
-            '[.[].failed_steps[] | test($pat; "i")] | any')
+            '[.[] | select([.failed_steps[] | test($pat; "i")] | any)] | length > 0')
+          HAS_INFRA_FAILURE=$(echo "$FAILED_JOBS" | jq --arg pat "$TEST_PATTERN" \
+            '[.[] | select((.failed_steps | length == 0) or ([.failed_steps[] | test($pat; "i")] | any | not))] | length > 0')
 
           echo "::endgroup::"
 
-          if [ "$HAS_TEST_FAILURE" = "true" ]; then
-            echo "::notice::Real test/lint failure detected — not retrying."
+          if [ "$HAS_INFRA_FAILURE" != "true" ]; then
+            echo "::notice::All failures are real test/lint failures — not retrying."
             exit 0
           fi
 
-          # All failures are infrastructure-related (setup, checkout, install, OOM, network)
-          echo "::notice::Only infrastructure failures detected — retrying (attempt $ATTEMPT → $((ATTEMPT + 1)))"
+          # There are infrastructure failures — retry.
+          # gh run rerun --failed will re-run all failed jobs; real test failures
+          # will fail again but infra failures get another chance.
+          echo "::notice::Infrastructure failures detected — retrying (attempt $ATTEMPT → $((ATTEMPT + 1)))"
 
           if [ "$HAS_CANCELLED" = "true" ]; then
-            # Sibling jobs were cancelled by cancel-on-failure step — must re-run entire workflow
+            # Sibling jobs were cancelled — must re-run entire workflow
             echo "Cancelled sibling jobs detected — re-running entire workflow."
             gh run rerun "$RUN_ID" --repo "$REPO"
           else
-            # Only specific jobs failed, others completed — re-run just the failed ones
+            # Re-run only failed jobs
             echo "Re-running only failed jobs."
             gh run rerun "$RUN_ID" --failed --repo "$REPO"
           fi

.github/workflows/ci.yaml

@@ -210,10 +210,11 @@ jobs:
       if: always()
       run: ccache -s
     - name: Run CPU tests
+      id: run-tests
       run: |
         pytest tests/cpu/ tests/contract/ -v --tb=short
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env:
@@ -291,10 +292,11 @@ jobs:
       run: |
         python -c "from candle._backends.mps.runtime import is_available; print('MPS available:', is_available())"
     - name: Run MPS tests
+      id: run-tests
       run: |
         pytest tests/mps/ -v --tb=short
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env:
@@ -348,13 +350,14 @@ jobs:
       if: always()
       run: ccache -s
     - name: Run NPU common + 910B op tests
+      id: run-tests
       run: |
         source /opt/miniconda3/etc/profile.d/conda.sh
         conda activate candle
         bash .github/scripts/ci-failfast-watcher.sh \
           pytest tests/npu/common/ tests/npu/910b/ -v --tb=short
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env:
@@ -406,6 +409,7 @@ jobs:
       if: always()
       run: ccache -s
     - name: Run NPU integration tests
+      id: run-tests
       run: |
         source /opt/miniconda3/etc/profile.d/conda.sh
         conda activate candle
@@ -417,7 +421,7 @@ jobs:
           --ignore=tests/npu/310b \
           --ignore=tests/npu/310p
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env:
@@ -470,13 +474,14 @@ jobs:
       if: always()
       run: ccache -s
     - name: Run HCCL distributed tests
+      id: run-tests
       run: |
         source /opt/miniconda3/etc/profile.d/conda.sh
         conda activate candle
         bash .github/scripts/ci-failfast-watcher.sh \
           pytest tests/distributed/ -v --tb=short -k "hccl"
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env:
@@ -569,6 +574,7 @@ jobs:
         }
         clean_env python -c "import candle as torch; count = torch.npu.device_count(); assert torch.npu.is_available(), 'expected NPU to be available in NPU Suite (6-7)'; assert count >= 2, f'expected >=2 visible NPUs, found {count}'"
     - name: Run NPU tests
+      id: run-tests
       run: |
         source "$ASCEND_ENV_SCRIPT"
         clean_env() {
@@ -602,7 +608,7 @@ jobs:
           --ignore=tests/npu/310b/ \
           --ignore=tests/npu/310p/
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env:
@@ -693,6 +699,7 @@ jobs:
         }
         clean_env python -c "import candle as torch; count = torch.npu.device_count(); assert torch.npu.is_available(), 'expected NPU to be available in Distributed Suite (4-5)'; assert count >= 2, f'expected >=2 visible NPUs, found {count}'"
     - name: Run distributed baseline suite
+      id: run-dist-tests
       run: |
         source "$ASCEND_ENV_SCRIPT"
         clean_env() {
@@ -722,6 +729,7 @@ jobs:
         clean_env bash .github/scripts/ci-failfast-watcher.sh \
           pytest tests/distributed/ -v --tb=short -k "not all_to_all_single_async_unequal_multicard and not all_to_all_single_invalid_split_pairing_multicard and not all_to_all_single_split_numel_validation_multicard and not test_ddp"
     - name: Run 2-card HCCL all_to_all coverage
+      id: run-hccl-tests
       run: |
         set -o pipefail
         source "$ASCEND_ENV_SCRIPT"
@@ -760,7 +768,7 @@ jobs:
           exit 1
         fi
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-dist-tests.outcome == 'failure' || steps.run-hccl-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env:
@@ -851,6 +859,7 @@ jobs:
         }
         clean_env python -c "import candle as torch; count = torch.npu.device_count(); assert torch.npu.is_available(), 'expected NPU to be available in HCCL 4-Card Suite (0-3)'; assert count >= 4, f'expected >=4 visible NPUs, found {count}'"
     - name: Run 4-card HCCL all_to_all coverage
+      id: run-hccl-4card-tests
       run: |
         set -o pipefail
         source "$ASCEND_ENV_SCRIPT"
@@ -889,7 +898,7 @@ jobs:
           exit 1
         fi
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-hccl-4card-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env:
@@ -1011,6 +1020,7 @@ jobs:
         }
         clean_env python -c "import candle as torch; assert torch.npu.is_available(), 'NPU not available on 310B runner'; print('NPU available:', torch.npu.is_available()); print('Device count:', torch.npu.device_count())"
     - name: Run 310B-specific tests
+      id: run-310b-tests
       run: |
         source "$ASCEND_ENV_SCRIPT"
         clean_env() {
@@ -1043,6 +1053,7 @@ jobs:
         sleep 2
         sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null || true
     - name: Run NPU common tests
+      id: run-common-tests
       run: |
         source "$ASCEND_ENV_SCRIPT"
         clean_env() {
@@ -1070,7 +1081,7 @@ jobs:
         clean_env bash .github/scripts/ci-failfast-watcher.sh \
           pytest tests/npu/common/ -v --tb=short
     - name: Cancel workflow on failure
-      if: failure()
+      if: steps.run-310b-tests.outcome == 'failure' || steps.run-common-tests.outcome == 'failure'
       run: |
         gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} || true
       env: