diff --git a/scripts/check_test_coverage.py b/scripts/check_test_coverage.py new file mode 100644 index 000000000..a384035f0 --- /dev/null +++ b/scripts/check_test_coverage.py @@ -0,0 +1,90 @@ +"""Fail if a test file matches none of the globs scripts/test.sh actually runs. + +A batch glob that names a numeric range (``test_006[0-1]*``) stops covering the +suite the moment someone adds ``test_0062``. That is how the whole +integration-point suite (#703, #707) and the swarm repopulation tests (#713) +shipped green without CI ever running them. The globs are still ranges — some +of them have to be — so this check is what keeps them honest. + +Files deferred on purpose are listed in DEFERRED with the reason and the issue. +The list may only shrink: adding to it needs a maintainer decision, exactly like +the allowlist in check_deprecated_patterns.py. +""" + +import glob +import re +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent + +# Deferred by maintainer decision, not by accident. +DEFERRED = { + # No issue: this band was disabled in scripts/test.sh as "potentially + # problematic" without one being filed. Recorded as it stands rather than + # dressed up — #721 follow-up work re-enables it and removes this entry. + "test_06[0-9][0-9]_*": "regression suite disabled in test.sh, no issue filed", + # Narrow: test_1072 is pulled out of this band and run by name, so a broad + # test_107* would list a covered file as deferred. + "test_1070_*": "level_2/level_3 + tier_b/tier_c, awaiting triage (#504)", + "test_1071_*": "level_2/level_3 + tier_b/tier_c, awaiting triage (#504)", + "test_1073_*": "level_2/level_3 + tier_b/tier_c, awaiting triage (#504)", + "test_106*": "level_2/level_3 + tier_b/tier_c, awaiting triage (#504)", +} + + +def globs_run_by(script): + """Every tests/... pattern passed to $PYTEST on a line that is not commented.""" + text = script.read_text().replace("\\\n", " ") + patterns = set() + for line in text.splitlines(): + if "$PYTEST" in line and not line.lstrip().startswith("#"): + patterns.update(re.findall(r"tests/[A-Za-z0-9_\[\]*.\-]+", line)) + return patterns + + +def main(): + covered = set() + for pattern in globs_run_by(REPO / "scripts" / "test.sh"): + covered.update(glob.glob(str(REPO / pattern))) + covered = {Path(p).name for p in covered} + + every = {p.name for p in (REPO / "tests").glob("test_*.py")} + + # A deferred entry that matches nothing, or matches only files a glob + # already runs, is stale. Without this the list only ever grows: an entry + # keeps reporting a deferral that stopped being true, which is how the + # shrink-only invariant quietly becomes a fiction. + deferred, stale = set(), [] + for pattern, reason in DEFERRED.items(): + matched = {Path(p).name for p in glob.glob(str(REPO / "tests" / pattern))} + if not matched: + stale.append(f"{pattern!r} matches no test file") + elif matched <= covered: + stale.append(f"{pattern!r} matches only files that already run") + deferred.update(matched) + + dark = sorted(every - covered - deferred) + + if dark: + print(f"{len(dark)} test file(s) match no glob in scripts/test.sh:") + for name in dark: + print(f" tests/{name}") + print("\nWiden the batch glob that should have caught them, or add an entry") + print("to DEFERRED in this file saying who deferred it and why.") + return 1 + + if stale: + print("DEFERRED has stale entries — the list may only shrink:") + for problem in stale: + print(f" {problem}") + print("\nRemove them: the files they name are running, or no longer exist.") + return 1 + + print(f"OK: all {len(every)} test files are reachable from scripts/test.sh " + f"({len(deferred)} deferred by decision).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/sessions/README.md b/scripts/sessions/README.md new file mode 100644 index 000000000..420bda165 --- /dev/null +++ b/scripts/sessions/README.md @@ -0,0 +1,15 @@ +# Session scripts + +Drivers, plotting scripts, benchmarks and profilers written during development +sessions. They are not tests and are not run by CI — they lived in `tests/` +until the 2026-09 audit, where 22 of them sat among the real test files and made +the test tree hard to read. + +Nothing imports these; they are run directly. Three of them +(`vep_fault_weakening.py`, `vep_strain_weakening.py`, `vep_timedep_yield.py`) +wrote their figures to an absolute path inside a worktree that no longer exists, +so they had been writing nowhere since that worktree was removed; they now write +beside themselves. + +A test helper belongs in `tests/` with a leading underscore (`_mg_ladder.py`), +not here. diff --git a/tests/benchmark_index_swarm_vectorized.py b/scripts/sessions/benchmark_index_swarm_vectorized.py similarity index 97% rename from tests/benchmark_index_swarm_vectorized.py rename to scripts/sessions/benchmark_index_swarm_vectorized.py index 33ba19018..132918175 100644 --- a/tests/benchmark_index_swarm_vectorized.py +++ b/scripts/sessions/benchmark_index_swarm_vectorized.py @@ -3,7 +3,7 @@ against the original loop-based reference. Uses a larger mesh to make the speedup measurable. -Run with: python tests/benchmark_index_swarm_vectorized.py +Run with: python scripts/sessions/benchmark_index_swarm_vectorized.py """ import numpy as np import underworld3 as uw diff --git a/tests/benchmark_projection_evaluate.py b/scripts/sessions/benchmark_projection_evaluate.py similarity index 100% rename from tests/benchmark_projection_evaluate.py rename to scripts/sessions/benchmark_projection_evaluate.py diff --git a/tests/minimal_vep_timing.py b/scripts/sessions/minimal_vep_timing.py similarity index 97% rename from tests/minimal_vep_timing.py rename to scripts/sessions/minimal_vep_timing.py index 797e34eb9..837676600 100644 --- a/tests/minimal_vep_timing.py +++ b/scripts/sessions/minimal_vep_timing.py @@ -1,6 +1,6 @@ """Minimal VEP timing test — isolate where time is spent. -Run with: pixi run -e amr-dev python tests/minimal_vep_timing.py +Run with: pixi run -e amr-dev python scripts/sessions/minimal_vep_timing.py """ import time diff --git a/tests/plot_ve_combined.py b/scripts/sessions/plot_ve_combined.py similarity index 99% rename from tests/plot_ve_combined.py rename to scripts/sessions/plot_ve_combined.py index 11e2a96b6..7ff9b0c3f 100644 --- a/tests/plot_ve_combined.py +++ b/scripts/sessions/plot_ve_combined.py @@ -5,7 +5,7 @@ - Right axis: strain rate (filled grey) Reads from saved .npz data — run the benchmarks first: - python tests/plot_ve_oscillatory_validation.py + python scripts/sessions/plot_ve_oscillatory_validation.py python docs/advanced/benchmarks/run_ve_square_wave.py """ diff --git a/tests/plot_ve_oscillatory_validation.py b/scripts/sessions/plot_ve_oscillatory_validation.py similarity index 97% rename from tests/plot_ve_oscillatory_validation.py rename to scripts/sessions/plot_ve_oscillatory_validation.py index 86d0683e7..95fcc85c6 100644 --- a/tests/plot_ve_oscillatory_validation.py +++ b/scripts/sessions/plot_ve_oscillatory_validation.py @@ -10,8 +10,8 @@ Results are saved as .npz checkpoint files for re-analysis. Usage: - python tests/plot_ve_oscillatory_validation.py - python tests/plot_ve_oscillatory_validation.py --replot # replot from saved data + python scripts/sessions/plot_ve_oscillatory_validation.py + python scripts/sessions/plot_ve_oscillatory_validation.py --replot # replot from saved data """ import numpy as np diff --git a/tests/plot_ve_vep_oscillatory.py b/scripts/sessions/plot_ve_vep_oscillatory.py similarity index 100% rename from tests/plot_ve_vep_oscillatory.py rename to scripts/sessions/plot_ve_vep_oscillatory.py diff --git a/tests/profile_jit_phases.py b/scripts/sessions/profile_jit_phases.py similarity index 98% rename from tests/profile_jit_phases.py rename to scripts/sessions/profile_jit_phases.py index c1c04a7da..ada8e3fec 100644 --- a/tests/profile_jit_phases.py +++ b/scripts/sessions/profile_jit_phases.py @@ -3,7 +3,7 @@ Instruments: sympy derivatives, expression unwrapping, hashing, C code generation, Cython compilation, and the actual PETSc solve. -Run with: pixi run -e default python tests/profile_jit_phases.py +Run with: pixi run -e default python scripts/sessions/profile_jit_phases.py """ import time diff --git a/tests/run_snapshot_backstepping_demo.py b/scripts/sessions/run_snapshot_backstepping_demo.py similarity index 98% rename from tests/run_snapshot_backstepping_demo.py rename to scripts/sessions/run_snapshot_backstepping_demo.py index 051fcd26c..668077b95 100644 --- a/tests/run_snapshot_backstepping_demo.py +++ b/scripts/sessions/run_snapshot_backstepping_demo.py @@ -21,7 +21,7 @@ visual point of the figure. Run: - pixi run -e amr-dev python tests/run_snapshot_backstepping_demo.py + pixi run -e amr-dev python scripts/sessions/run_snapshot_backstepping_demo.py Output: snapshot_backstepping_demo.png in the current working directory. diff --git a/tests/run_snapshot_backstepping_spatial.py b/scripts/sessions/run_snapshot_backstepping_spatial.py similarity index 98% rename from tests/run_snapshot_backstepping_spatial.py rename to scripts/sessions/run_snapshot_backstepping_spatial.py index fe06061f5..060816bd8 100644 --- a/tests/run_snapshot_backstepping_spatial.py +++ b/scripts/sessions/run_snapshot_backstepping_spatial.py @@ -21,7 +21,7 @@ over-stretched, CFL violated) vs. ten substeps at sub-CFL Δt. Run: - pixi run -e amr-dev python tests/run_snapshot_backstepping_spatial.py + pixi run -e amr-dev python scripts/sessions/run_snapshot_backstepping_spatial.py Output: snapshot_backstepping_spatial.png in the current working directory. diff --git a/tests/run_ve_order2_debug.py b/scripts/sessions/run_ve_order2_debug.py similarity index 100% rename from tests/run_ve_order2_debug.py rename to scripts/sessions/run_ve_order2_debug.py diff --git a/tests/run_ve_oscillatory.py b/scripts/sessions/run_ve_oscillatory.py similarity index 100% rename from tests/run_ve_oscillatory.py rename to scripts/sessions/run_ve_oscillatory.py diff --git a/tests/run_ve_shear_order2_quick.py b/scripts/sessions/run_ve_shear_order2_quick.py similarity index 100% rename from tests/run_ve_shear_order2_quick.py rename to scripts/sessions/run_ve_shear_order2_quick.py diff --git a/tests/run_ve_shear_quick.py b/scripts/sessions/run_ve_shear_quick.py similarity index 100% rename from tests/run_ve_shear_quick.py rename to scripts/sessions/run_ve_shear_quick.py diff --git a/tests/run_ve_shear_validation.py b/scripts/sessions/run_ve_shear_validation.py similarity index 97% rename from tests/run_ve_shear_validation.py rename to scripts/sessions/run_ve_shear_validation.py index db2195054..6c9b10975 100644 --- a/tests/run_ve_shear_validation.py +++ b/scripts/sessions/run_ve_shear_validation.py @@ -1,6 +1,6 @@ """Quick validation script for the VE shear box test. -Run with: pixi run -e amr-dev python tests/run_ve_shear_validation.py +Run with: pixi run -e amr-dev python scripts/sessions/run_ve_shear_validation.py """ import time as timer diff --git a/tests/run_ve_vep_oscillatory_checkpoint.py b/scripts/sessions/run_ve_vep_oscillatory_checkpoint.py similarity index 100% rename from tests/run_ve_vep_oscillatory_checkpoint.py rename to scripts/sessions/run_ve_vep_oscillatory_checkpoint.py diff --git a/tests/run_ve_vep_oscillatory_plot.py b/scripts/sessions/run_ve_vep_oscillatory_plot.py similarity index 100% rename from tests/run_ve_vep_oscillatory_plot.py rename to scripts/sessions/run_ve_vep_oscillatory_plot.py diff --git a/tests/run_vep_oscillatory.py b/scripts/sessions/run_vep_oscillatory.py similarity index 100% rename from tests/run_vep_oscillatory.py rename to scripts/sessions/run_vep_oscillatory.py diff --git a/tests/run_vep_shear_box.py b/scripts/sessions/run_vep_shear_box.py similarity index 100% rename from tests/run_vep_shear_box.py rename to scripts/sessions/run_vep_shear_box.py diff --git a/tests/run_vp_shear_box.py b/scripts/sessions/run_vp_shear_box.py similarity index 100% rename from tests/run_vp_shear_box.py rename to scripts/sessions/run_vp_shear_box.py diff --git a/tests/vep_fault_weakening.py b/scripts/sessions/vep_fault_weakening.py similarity index 96% rename from tests/vep_fault_weakening.py rename to scripts/sessions/vep_fault_weakening.py index e186ce77b..7d95190d1 100644 --- a/tests/vep_fault_weakening.py +++ b/scripts/sessions/vep_fault_weakening.py @@ -3,12 +3,13 @@ Horizontal fault at y=0.5 using Surface gaussian influence function. Runs at two vertical resolutions to check convergence. -Run: pixi run -e amr-dev python tests/vep_fault_weakening.py +Run: pixi run -e amr-dev python scripts/sessions/vep_fault_weakening.py """ import time import numpy as np import sympy +import os import underworld3 as uw ETA = 1.0 @@ -153,6 +154,6 @@ def run_fault_model(res_x, res_y): fig.suptitle(f"VEP embedded fault convergence: $\\tau_y$={TAU_Y_FAULT}/{TAU_Y_BULK}, width={FAULT_WIDTH}", fontsize=12, y=1.02) fig.tight_layout() -out_path = "/Users/lmoresi/+Underworld/underworld3-pixi/.claude/worktrees/solver-unification/vep_fault.png" +out_path = os.path.join(os.path.dirname(__file__), "vep_fault.png") fig.savefig(out_path, dpi=150, bbox_inches='tight') uw.pprint(0, f"Saved {out_path}") diff --git a/tests/vep_strain_weakening.py b/scripts/sessions/vep_strain_weakening.py similarity index 96% rename from tests/vep_strain_weakening.py rename to scripts/sessions/vep_strain_weakening.py index db6c3ae26..ee4b45fdf 100644 --- a/tests/vep_strain_weakening.py +++ b/scripts/sessions/vep_strain_weakening.py @@ -8,12 +8,13 @@ No evaluate, no projection — pure numpy on stored data. -Run: pixi run -e amr-dev python tests/vep_strain_weakening.py +Run: pixi run -e amr-dev python scripts/sessions/vep_strain_weakening.py """ import time import numpy as np import sympy +import os import underworld3 as uw ETA = 1.0 @@ -145,6 +146,6 @@ axes[2].grid(True, alpha=0.3) fig.tight_layout() -out_path = "/Users/lmoresi/+Underworld/underworld3-pixi/.claude/worktrees/solver-unification/vep_strain_weakening.png" +out_path = os.path.join(os.path.dirname(__file__), "vep_strain_weakening.png") fig.savefig(out_path, dpi=150) print(f"Saved {out_path}") diff --git a/tests/vep_timedep_yield.py b/scripts/sessions/vep_timedep_yield.py similarity index 95% rename from tests/vep_timedep_yield.py rename to scripts/sessions/vep_timedep_yield.py index 4b9cc5af7..090d25b27 100644 --- a/tests/vep_timedep_yield.py +++ b/scripts/sessions/vep_timedep_yield.py @@ -6,12 +6,13 @@ No strain accumulation, no projection of viscosity ratios — just the solver responding to a changing yield stress parameter. -Run: pixi run -e amr-dev python tests/vep_timedep_yield.py +Run: pixi run -e amr-dev python scripts/sessions/vep_timedep_yield.py """ import time import numpy as np import sympy +import os import underworld3 as uw ETA = 1.0 @@ -132,6 +133,6 @@ ax.grid(True, alpha=0.3) fig.tight_layout() -out_path = "/Users/lmoresi/+Underworld/underworld3-pixi/.claude/worktrees/solver-unification/vep_timedep_yield.png" +out_path = os.path.join(os.path.dirname(__file__), "vep_timedep_yield.png") fig.savefig(out_path, dpi=150) print(f"Saved {out_path}") diff --git a/scripts/test.sh b/scripts/test.sh index 6dc40415b..953d24b6a 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -91,6 +91,12 @@ if [ $PARALLEL_ONLY -eq 0 ]; then echo "Running serial test suite..." echo "" + # Every test file must be reachable from a glob below. #721 was a range that + # stopped covering its own band; 586224f6 took 006x-007x whole so that one + # cannot recur, but the other ranges are still hand-maintained. This is what + # notices when one of them goes stale. + python3 "$(dirname "$0")/check_test_coverage.py" || status=1 + # Run simple tests (0000-0299: basic functionality, imports, simple operations) $PYTEST tests/test_00[0-4]*py || status=1 $PYTEST tests/test_0050*py || status=1 @@ -102,7 +108,7 @@ if [ $PARALLEL_ONLY -eq 0 ]; then # siblings is not dark again. Verified passing (103 tests) before wiring in. $PYTEST tests/test_005[1-9]*py tests/test_00[6-7]*py || status=1 $PYTEST tests/test_01*py || status=1 - $PYTEST tests/test_02*py || status=1 + $PYTEST tests/test_02*py tests/test_03*py || status=1 # Intermediate tests (0500-0799: data structures, transformations, enhanced interfaces) # NOTE: Temporarily disabling test_06*py regression tests (potentially problematic) @@ -113,7 +119,7 @@ if [ $PARALLEL_ONLY -eq 0 ]; then $PYTEST tests/test_08*py || status=1 # Poisson solvers (including Darcy flow) - $PYTEST tests/test_100[0-9]*py || status=1 + $PYTEST tests/test_100[0-9]*py tests/test_103*py || status=1 # Solver / system tests (advanced solver problems) # test_101* / test_102* include the rotated free-slip suite (test_1018, diff --git a/vep_fault.png b/vep_fault.png deleted file mode 100644 index 690ffba4b..000000000 Binary files a/vep_fault.png and /dev/null differ diff --git a/vep_strain_weakening.png b/vep_strain_weakening.png deleted file mode 100644 index 66cf748cb..000000000 Binary files a/vep_strain_weakening.png and /dev/null differ diff --git a/vep_timedep_yield.png b/vep_timedep_yield.png deleted file mode 100644 index 0203648f4..000000000 Binary files a/vep_timedep_yield.png and /dev/null differ