diff --git a/.github/workflows/build_uw3_and_test.yaml b/.github/workflows/build_uw3_and_test.yaml index e7286f99a..d049cedba 100644 --- a/.github/workflows/build_uw3_and_test.yaml +++ b/.github/workflows/build_uw3_and_test.yaml @@ -45,13 +45,16 @@ env: jobs: test: runs-on: ubuntu-latest - # The suite has outgrown 60 minutes. Measured 2026-08-15: the SERIAL phase - # alone is ~55 min (three batches at 14-15 min each), so the parallel phase - # - which runs last - was being cut off mid-run and reported as a failure - # that looked like a test failure. Any addition at all pushed a run over. - # This is a stop-gap that unblocks landing work; the durable fix is to split - # the batches across parallel jobs so wall-clock stops tracking total test - # time. See the CI-runtime issue for that. + # Raised past 60 minutes on a 2026-08-15 measurement (serial phase ~55 min). + # xdist has since cut that: the five successful runs before 2026-09-12 spent + # 34, 35, 43, 44 and 48 minutes in "Run tests", well inside this budget. The + # headroom is why the previously-deferred bands could simply be switched on + # (#721 follow-up) rather than traded against something else. + # + # Sharding the batches across parallel jobs remains the durable fix if this + # tightens again; it was referenced here as "the CI-runtime issue" but never + # actually filed, so the numbers above are recorded in the issue that this + # change opens. timeout-minutes: 120 steps: diff --git a/scripts/check_test_coverage.py b/scripts/check_test_coverage.py index a384035f0..514f89f39 100644 --- a/scripts/check_test_coverage.py +++ b/scripts/check_test_coverage.py @@ -20,16 +20,11 @@ # Deferred by maintainer decision, not by accident. DEFERRED = { - # No issue: this band was disabled in scripts/test.sh as "potentially - # problematic" without one being filed. Recorded as it stands rather than - # dressed up — #721 follow-up work re-enables it and removes this entry. - "test_06[0-9][0-9]_*": "regression suite disabled in test.sh, no issue filed", - # Narrow: test_1072 is pulled out of this band and run by name, so a broad - # test_107* would list a covered file as deferred. - "test_1070_*": "level_2/level_3 + tier_b/tier_c, awaiting triage (#504)", - "test_1071_*": "level_2/level_3 + tier_b/tier_c, awaiting triage (#504)", - "test_1073_*": "level_2/level_3 + tier_b/tier_c, awaiting triage (#504)", - "test_106*": "level_2/level_3 + tier_b/tier_c, awaiting triage (#504)", + # Empty, and that is the point. Every band that used to sit here — the + # test_06NN regression suite and test_106*/test_107* — is now batched in + # scripts/test.sh. Nothing is excluded from CI by its number any more; a + # test that should not gate says so with @pytest.mark.tier_c, which is a + # property of the test rather than of where it sits in the numbering. } @@ -38,7 +33,10 @@ def globs_run_by(script): text = script.read_text().replace("\\\n", " ") patterns = set() for line in text.splitlines(): - if "$PYTEST" in line and not line.lstrip().startswith("#"): + # Both spellings: the runner is a bash ARRAY, so call sites read + # "${PYTEST[@]}", but a plain $PYTEST is still worth matching in case + # one is left behind or reintroduced. + if re.search(r"\$\{?PYTEST", line) and not line.lstrip().startswith("#"): patterns.update(re.findall(r"tests/[A-Za-z0-9_\[\]*.\-]+", line)) return patterns diff --git a/scripts/test.sh b/scripts/test.sh index 953d24b6a..3baceb943 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -78,14 +78,33 @@ export MKL_NUM_THREADS=1 # # Unset (or 1) still runs everything in one process, which is what you want # when a test passes alone and fails in a full run. +# An ARRAY, not a string. Every call site below expands it unquoted, so a string +# would be word-split by bash: `-m 'not tier_c'` becomes the two words `'not` and +# `tier_c'`, and pytest answers that by silently collecting NOTHING and exiting 0 +# — a green run that tested nothing. An array carries its elements intact through +# "${PYTEST[@]}", and globs on the command line still expand normally. +PYTEST=(pytest --config-file=tests/pytest.ini) if [ -n "$WORKERS" ] && [ "$WORKERS" -gt 1 ]; then echo "Serial batches: $WORKERS worker process(es)" - PYTEST="pytest --config-file=tests/pytest.ini --dist loadfile -n $WORKERS" + PYTEST+=(--dist loadfile -n "$WORKERS") else echo "Serial batches: in-process (set WORKERS=N to distribute)" - PYTEST="pytest --config-file=tests/pytest.ini" fi +# Tier C never gates. A tier C failure demands an EXPLANATION, not a revert +# (Charter S8, docs/developer/TESTING-RELIABILITY-SYSTEM.md): these are +# characterisations that can fail because the code got BETTER, and comparisons +# whose result moves with the fixture. They are excluded from the batches that +# set $status and run in their own reporting pass at the end, which is read but +# cannot fail the build. +# Tier C never gates. A tier C failure demands an EXPLANATION, not a revert +# (Charter S8, docs/developer/TESTING-RELIABILITY-SYSTEM.md): these are +# characterisations that can fail because the code got BETTER, and comparisons +# whose result moves with the fixture. They are excluded from the batches that +# set $status and run in their own reporting pass at the end, which is read but +# cannot fail the build. +PYTEST+=(-m "not tier_c") + # Run serial tests (unless --parallel-only specified) if [ $PARALLEL_ONLY -eq 0 ]; then echo "Running serial test suite..." @@ -98,35 +117,37 @@ if [ $PARALLEL_ONLY -eq 0 ]; then python3 "$(dirname "$0")/check_test_coverage.py" || status=1 # Run simple tests (0000-0299: basic functionality, imports, simple operations) - $PYTEST tests/test_00[0-4]*py || status=1 - $PYTEST tests/test_0050*py || status=1 + "${PYTEST[@]}" tests/test_00[0-4]*py || status=1 + "${PYTEST[@]}" tests/test_0050*py || status=1 # test_006[2-9] and test_007x matched NO batch glob and so never ran in CI: # the whole integration-point suite (0064-0067), swarm repopulation, # mid-time velocity and the VE stress history. They are not covered by the # disabled test_06*py line below either - that one is 0600-0699. The band is # taken whole (00[6-7]) rather than enumerated, so a test added next to its # siblings is not dark again. Verified passing (103 tests) before wiring in. - $PYTEST tests/test_005[1-9]*py tests/test_00[6-7]*py || status=1 - $PYTEST tests/test_01*py || status=1 - $PYTEST tests/test_02*py tests/test_03*py || status=1 + "${PYTEST[@]}" tests/test_005[1-9]*py tests/test_00[6-7]*py || status=1 + "${PYTEST[@]}" tests/test_01*py || status=1 + "${PYTEST[@]}" tests/test_02*py tests/test_03*py || status=1 # Intermediate tests (0500-0799: data structures, transformations, enhanced interfaces) - # NOTE: Temporarily disabling test_06*py regression tests (potentially problematic) - $PYTEST tests/test_05*py tests/test_07*py || status=1 - # $PYTEST tests/test_06*py || status=1 # DISABLED - regression tests need validation + # test_06*py was disabled as "potentially problematic" and stayed that way. The + # whole band was measured 2026-09-12 and passes; a band is not a unit of trust, + # and holding one back for a suspicion nobody recorded meant its level_1 files + # never ran at all. + "${PYTEST[@]}" tests/test_05*py tests/test_06*py tests/test_07*py || status=1 # Units system tests (0800-0899: unit-aware functions, arrays, and conversions) - $PYTEST tests/test_08*py || status=1 + "${PYTEST[@]}" tests/test_08*py || status=1 # Poisson solvers (including Darcy flow) - $PYTEST tests/test_100[0-9]*py tests/test_103*py || status=1 + "${PYTEST[@]}" tests/test_100[0-9]*py tests/test_103*py || status=1 # Solver / system tests (advanced solver problems) # test_101* / test_102* include the rotated free-slip suite (test_1018, # issue #504) and the MG / boundary-flux suites, which previously matched # no batch glob and never ran in CI. - $PYTEST tests/test_101*py tests/test_102*py || status=1 - $PYTEST tests/test_105*py || status=1 + "${PYTEST[@]}" tests/test_101*py tests/test_102*py || status=1 + "${PYTEST[@]}" tests/test_105*py || status=1 # The boundary-normal guard lives under tests/parallel/ but carries NO # mpi(min_size=2) mark, because the defect it guards is present in SERIAL in @@ -134,26 +155,27 @@ if [ $PARALLEL_ONLY -eq 0 ]; then # spherical shell at np=1). Run it here so the serial job covers that path — # every other serial test of the default boundary normal is on a box, where # flat walls make the question vacuous. It also runs in the --p N batch below. - $PYTEST tests/parallel/test_1069_boundary_normal_parallel.py || status=1 - # NOT yet batched (issue #504 audit): test_106*py and test_107*py contain - # level_2/level_3 + slow + tier_b/tier_c suites (e.g. test_1064) and need - # a triage/deselect decision before being wired into CI. + "${PYTEST[@]}" tests/parallel/test_1069_boundary_normal_parallel.py || status=1 + # test_106*/test_107* were held back pending "a triage/deselect decision". That + # decision is made: the band is not the unit of exclusion. Of the 243 tests in + # the deferred bands only 26 are level_3 or tier_c; 55 are level_1 AND tier_a — + # hardened, fast, and exactly what CI exists to run. test_1064 is the slow one + # and it is excluded by WHAT IT IS (tier_c) below, not by its number. + "${PYTEST[@]}" tests/test_106*py tests/test_107*py || status=1 # - # test_1072 is pulled forward out of that group, the same way test_1069 is - # above, because leaving it there defeats its purpose: it is the ONLY guard on - # the 3-D free-surface sign and relaxation rate, and #496 exists precisely - # because those regressions were invisible to CI. Landing the test into an - # unbatched file would have closed the issue without closing the gap. - # level_2/tier_b, ~55s serial; passes at np=1 and np=2. - $PYTEST tests/test_1072_free_surface_spherical.py || status=1 + # test_1072 used to be pulled forward out of test_107* by name, because that + # band was not batched and it is the ONLY guard on the 3-D free-surface sign + # and relaxation rate (#496 exists because those regressions were invisible to + # CI). The band runs as a whole now, so the named line is gone rather than + # running it twice. # Diffusion / Advection tests - $PYTEST tests/test_1100*py || status=1 - $PYTEST tests/test_1110*py tests/test_1120*py || status=1 # Annulus + vector SL - $PYTEST tests/test_1450*py || status=1 + "${PYTEST[@]}" tests/test_1100*py || status=1 + "${PYTEST[@]}" tests/test_1110*py tests/test_1120*py || status=1 # Annulus + vector SL + "${PYTEST[@]}" tests/test_1450*py || status=1 # Named (un-numbered) test files - JIT, docstrings, projections - $PYTEST tests/test_docstring_utils.py tests/test_jit_cache.py \ + "${PYTEST[@]}" tests/test_docstring_utils.py tests/test_jit_cache.py \ tests/test_jit_deterministic_ordering.py \ tests/test_multicomponent_projection.py \ tests/test_snes_vector_asymmetric_jacobian.py \ @@ -249,6 +271,23 @@ else echo "⚠️ Skipping parallel tests (use --p N to enable)" fi +# Tier C: run and report, never gate. Failures here are read by a human and +# answered with an explanation or a re-characterisation, never with a revert. +if [ $PARALLEL_ONLY -eq 0 ]; then + echo "" + echo "==========================================" + echo "Tier C characterisations (reported, not gating)" + echo "==========================================" + if pytest --config-file=tests/pytest.ini -m tier_c tests/; then + echo "Tier C: all characterisations still hold." + else + echo "" + echo "⚠️ A tier C characterisation changed. This does NOT fail the build." + echo " Explain what moved and re-characterise the test — do not revert" + echo " code to make it pass. See docs/developer/TESTING-RELIABILITY-SYSTEM.md." + fi +fi + # if [ $status -ne 0 ]; then echo ""