Run the test suite in parallel and gate on the combined coverage

.github/actions/load-shared-vars/action.yml

@@ -19,6 +19,9 @@ outputs:
   min-deps-matrix:
     description: "Matrix object for the minimum dependency test job"
     value: ${{ steps.load.outputs.min-deps-matrix }}
+  min-deps-python-default:
+    description: "The min-deps cell which runs the steps only one cell needs"
+    value: ${{ steps.load.outputs.min-deps-python-default }}
   network-os-matrix:
     description: "OS list for the network test job"
     value: ${{ steps.load.outputs.network-os-matrix }}
@@ -37,6 +40,9 @@ runs:
         # matrix (each entry there costs one environment cache per OS).
         full_py='["3.12","3.13","3.14"]'
         min_deps_py='["3.12","3.14"]'
+        # The one min-deps cell that runs the doctests: they take minutes and
+        # say the same thing on every version.
+        min_deps_py_default="3.12"
 
         # What a pull request runs. Every python version and every OS is
         # still covered; only the redundant cross product is dropped. The
@@ -60,4 +66,5 @@ runs:
         echo "python-default=$python_default" >> "$GITHUB_OUTPUT"
         echo "test-matrix=$test_matrix" >> "$GITHUB_OUTPUT"
         echo "min-deps-matrix=$min_deps_matrix" >> "$GITHUB_OUTPUT"
+        echo "min-deps-python-default=$min_deps_py_default" >> "$GITHUB_OUTPUT"
         echo "network-os-matrix=$network_os" >> "$GITHUB_OUTPUT"

.github/test_code.sh

@@ -1,22 +1,39 @@
 #!/bin/bash
 
-# Script to run tests to account for wonkiness of periodic mac failures.
-args=(tests -m "not network" -s --cov dascore --cov-append --cov-report=xml)
+# Runs one flavor of the test suite. Coverage is written to a data file, not
+# xml: each CI cell keeps its own file and the coverage_gate job combines
+# them, because no single OS covers every line (see runtests.yml).
+
+# sysmon is coverage's sys.monitoring core, ~1.11x the no-coverage runtime
+# against ~1.67x for the C tracer. It needs python >= 3.12 (the floor) and
+# does not support branch coverage, which is off here.
+export COVERAGE_CORE=sysmon
+
+# -n logical rather than -n auto: xdist's auto asks psutil for *physical*
+# cores, which is 2 on the SMT-enabled runners; logical gives all 3-4.
+parallel=(-n logical --dist loadfile)
+cov_args=(--cov dascore --cov-append --cov-report=)
+
+args=(tests -m "not network" "${parallel[@]}" "${cov_args[@]}")
 if [[ "$1" == "network" ]]; then
-  args=(tests -m network -s --cov dascore --cov-append --cov-report=xml)
+  args=(tests -m network "${parallel[@]}" "${cov_args[@]}")
 fi
 if [[ "$1" == "doctest" ]]; then
   args=(dascore --doctest-modules)
 fi
 if [[ "$1" == "profile" ]]; then
+  # No xdist: codspeed measures this process.
   args=(benchmarks --codspeed)
 fi
 
-exit_code=0
+python -c "
+import os
+try:
+    import psutil
+    physical = psutil.cpu_count(logical=False)
+except ImportError:
+    physical = None
+print(f'cpus: logical={os.cpu_count()} physical={physical}')
+"
 
-python -m pytest "${args[@]}" || exit_code=$?
-
-# Check the exit code is related to sporadic failures on mac, see #312
-if [ $exit_code -ne 132 ] && [ $exit_code -ne 0 ]; then
-  exit $exit_code
-fi
+python -m pytest "${args[@]}"

.github/workflows/run_min_dep_tests.yml

@@ -43,6 +43,7 @@ jobs:
     outputs:
       # Shared values live in .github/actions/load-shared-vars/action.yml
       min-deps-matrix: ${{ steps.load-vars.outputs.min-deps-matrix }}
+      min-deps-python-default: ${{ steps.load-vars.outputs.min-deps-python-default }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
@@ -97,9 +98,12 @@ jobs:
         shell: bash
         run: ./.github/test_code.sh
 
-      # Runs examples in docstrings
+      # Runs examples in docstrings. One cell: without numba the jit kernels'
+      # examples run in pure python, which costs minutes on every OS and
+      # version alike.
       - name: test docstrings
         id: run_docstrings
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == needs.setup.outputs.min-deps-python-default
         continue-on-error: ${{ env.debug_enabled == 'true' }}
         shell: bash
         run: ./.github/test_code.sh doctest

.github/workflows/runtests.yml

@@ -81,6 +81,8 @@ jobs:
     env:
       debug_enabled: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'debug') }}
       PYTEST_ADDOPTS: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'debug') && '-vv --durations=100' || '' }}
+      # One data file per cell; the coverage_gate job combines them all.
+      COVERAGE_FILE: .coverage.${{ matrix.os }}-${{ matrix.python-version }}
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
@@ -94,8 +96,14 @@ jobs:
           python-version: ${{ matrix.python-version }}
           prepare-test-data: "true"
 
+      # The generated docs tests and the doctests are the same on every cell
+      # and cost minutes, so one cell runs each. The generated tests run
+      # inside the measured suite below, so their coverage reaches the gate;
+      # the doctests are not measured, and counting them would let a line
+      # reachable only from a docstring example satisfy the gate.
       - name: generate qmd docs tests
         id: generate_qmd_tests
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == needs.setup.outputs.python-default
         shell: bash
         run: python scripts/generate_doc_code_tests.py
 
@@ -110,6 +118,7 @@ jobs:
       # Runs examples in docstrings
       - name: test docstrings
         id: run_docstrings
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == needs.setup.outputs.python-default
         continue-on-error: ${{ env.debug_enabled == 'true' }}
         shell: bash
         run: ./.github/test_code.sh doctest
@@ -129,19 +138,80 @@ jobs:
         with:
           limit-access-to-actor: true
 
-      # Upload coverage files
+      # The cell's coverage data, for the coverage_gate job to combine.
+      # Uploaded even when the tests failed, so a partial report is still
+      # readable; the gate then fails on the missing lines rather than on a
+      # missing artifact.
+      - name: upload coverage data
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: coverage-${{ matrix.os }}-${{ matrix.python-version }}
+          # A coverage data file starts with a dot.
+          path: ${{ env.COVERAGE_FILE }}
+          include-hidden-files: true
+          if-no-files-found: error
+
+      - name: fail job after debug session if tests failed
+        if: steps.generate_qmd_tests.outcome == 'failure' || steps.run_test_suite.outcome == 'failure' || steps.run_docstrings.outcome == 'failure'
+        shell: bash
+        run: exit 1
+
+  # Where the 100% coverage threshold is enforced. It has to run on the
+  # combined data rather than in any one cell: a handful of lines are
+  # reachable only on a case-insensitive filesystem (macOS, Windows), and
+  # the generated docs tests run on one cell.
+  coverage_gate:
+    needs: [setup, test_code]
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+
+    # Report on whatever the cells produced, including when one of them
+    # failed; do not wait on network_tests, which is allowed to fail.
+    if: ${{ !cancelled() && (github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci')) }}
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+
+      # Coverage alone: this job reads the source tree, it does not import
+      # dascore, so the dependency install is not worth its minute.
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: ${{ needs.setup.outputs.python-default }}
+
+      - name: install coverage
+        shell: bash
+        run: python -m pip install "coverage>=7.4,<8"
+
+      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131  # v7.0.0
+        with:
+          pattern: coverage-*
+          merge-multiple: true
+
+      # [tool.coverage.paths] in pyproject.toml is what lets the three
+      # operating systems' data files combine into one set of files.
+      - name: combine coverage
+        shell: bash
+        run: |
+          coverage combine
+          coverage report --show-missing
+          coverage xml
+
       - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
+        if: ${{ !cancelled() }}
         with:
-          fail_ci_if_error: false
+          fail_ci_if_error: true
           files: ./coverage.xml
           flags: unittests
-          name: PR_tests
+          name: combined_tests
           token: ${{ secrets.CODECOV_TOKEN }}
 
-      - name: fail job after debug session if tests failed
-        if: steps.generate_qmd_tests.outcome == 'failure' || steps.run_test_suite.outcome == 'failure' || steps.run_docstrings.outcome == 'failure'
+      # Last, so the report is uploaded whether or not this passes.
+      - name: require full coverage
         shell: bash
-        run: exit 1
+        run: coverage report --fail-under=100
 
   network_tests:
     needs: setup
@@ -157,6 +227,9 @@ jobs:
     # Keep remote-IO coverage visible without blocking unrelated changes.
     if: github.event_name == 'push' || !contains(github.event.pull_request.labels.*.name, 'no_ci')
 
+    env:
+      COVERAGE_FILE: .coverage.network-${{ matrix.os }}
+
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
@@ -173,7 +246,19 @@ jobs:
         shell: bash
         run: ./.github/test_code.sh network
 
+      # One network report per run, from one OS: the codecov comment waits
+      # for a fixed number of uploads (codecov.yml), and the three operating
+      # systems exercise the same remote code.
+      # !cancelled(), not the implicit success(): the tests here are allowed
+      # to fail, and skipping the xml would leave codecov waiting forever for
+      # the second of the two uploads it counts.
+      - name: write coverage xml
+        if: ${{ matrix.os == 'ubuntu-latest' && !cancelled() }}
+        shell: bash
+        run: coverage xml
+
       - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
+        if: ${{ matrix.os == 'ubuntu-latest' && !cancelled() }}
         with:
           fail_ci_if_error: false
           files: ./coverage.xml
@@ -209,14 +294,21 @@ jobs:
       - uses: ./.github/actions/mamba-install-dascore
         with:
           python-version: ${{ needs.setup.outputs.python-default }}
-          # [test] rather than the [dev] default: pip would otherwise layer
-          # ~30 PyPI distributions over the conda environment, which is the
-          # opposite of what this job is checking.
-          install-group-str: "[test]"
+          # Installed below with --no-deps instead: letting pip resolve the
+          # dependencies would layer PyPI distributions over the conda
+          # environment and so hide whatever environment.yml is missing.
+          install-package: "false"
           prepare-test-data: "true"
 
-      # No coverage upload: the uv jobs already cover this suite, and this job
-      # exists to prove environment.yml still solves and dascore works in it.
-      - name: run test suite
+      - name: install dascore on the conda environment alone
+        shell: bash -el {0}
+        run: |
+          pip install --no-deps -e .
+          pip check
+
+      # A smoke test, not the suite: every test already runs on the uv
+      # matrix, and what this job answers is whether environment.yml still
+      # solves and dascore reads a file in the environment it describes.
+      - name: run a subset of the test suite
         shell: bash -el {0}
-        run: python -m pytest tests -m "not network" -q
+        run: python -m pytest tests/test_io/test_dasdae tests/test_compat.py -q

.github/workflows/test_free_threaded.yml

@@ -60,7 +60,7 @@ jobs:
       - name: install dascore
         run: |
           python -m pip install --upgrade pip
-          pip install -e . pytest pytest-timeout
+          pip install -e . pytest pytest-timeout pytest-xdist
 
       - name: confirm the GIL is disabled
         run: |
@@ -74,4 +74,4 @@ jobs:
         uses: ./.github/actions/prime-test-data-cache
 
       - name: run test suite without the GIL
-        run: python -m pytest tests -m "not network" -q --timeout=900
+        run: python -m pytest tests -m "not network" -q --timeout=900 -n logical --dist loadfile

codecov.yml

@@ -1,9 +1,12 @@
+# Two uploads reach codecov per run: one `unittests` report from the
+# coverage_gate job (every test cell combined) and one `network` report from
+# the ubuntu network job.
 codecov:
   notify:
-    after_n_builds: 7
+    after_n_builds: 2
 
 comment:
-  after_n_builds: 7
+  after_n_builds: 2
 
 # Require 100% coverage on the unittests flag.
 coverage:

docs/contributing/testing.qmd

@@ -20,6 +20,18 @@ introduce large blocks of dead code.
 pytest tests --cov dascore --cov-report term-missing
 ```
 
+The suite is parallel safe, and CI runs it that way. Locally it is the
+quickest way to run the whole thing:
+
+<!--pytest-codeblocks:skip-->
+```bash
+pytest tests -n logical --dist loadfile
+```
+
+CI requires 100% coverage of the combined data from every operating system
+rather than from any one run, so a Linux-only run can report a line or two
+missing which macOS and Windows cover.
+
 If you would like to test the IO modules it can be done like so:
 
 <!--pytest-codeblocks:skip-->