Add a --regenerate-snapshots pytest option

conftest.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Copyright © 2026 SparkMeter, Inc.
+# All Rights Reserved.
+"""Top level pytest configuration.
+
+This file exists at the top of the checkout, rather than alongside the tests
+in sparkmeter/conftest.py, because pytest only calls pytest_addoption for
+*initial* conftests -- the ones found by walking down to the invocation's
+anchor directory before the command line is parsed. With no path argument the
+anchor is the rootdir, so a conftest.py one directory further down is loaded
+too late for its options to exist: `pytest --regenerate-snapshots` would be
+rejected as an unrecognized argument.
+"""
+
+# pytest's own `pytester` fixture, used by the tests that run pytest on a
+# throwaway test to check this file's wiring end to end. `pytest_plugins` is
+# only honored in the top level conftest, which is here.
+pytest_plugins = ["pytester"]
+
+
+def pytest_addoption(parser):
+    """Register our own command line options."""
+    parser.addoption(
+        "--regenerate-snapshots",
+        action="store_true",
+        default=False,
+        help=(
+            "Rewrite the .page snapshot of any content test whose rendered "
+            "output differs, instead of failing the test."
+        ),
+    )
+
+
+def pytest_configure(config):
+    """Propagate command line options to the code that needs them.
+
+    ContentTester is a plain class with no fixture access, so the option is
+    handed to it as a module level flag. This runs once per process, which
+    covers each xdist worker as well as a plain single-process run.
+    """
+    from sparkmeter.web import unittestutils
+
+    unittestutils.regenerate_snapshots = config.getoption("--regenerate-snapshots")

deploy/Dockerfile

@@ -169,6 +169,12 @@ USER spark
 
 COPY --chown=spark:spark setup.cfg setup.cfg
 
+# The top level conftest.py, which registers the suite's own command line
+# options. It has to be at the top of the tree, next to setup.cfg: pytest only
+# calls pytest_addoption for initial conftests, and with no path argument the
+# rootdir is the only directory searched for one.
+COPY --chown=spark:spark conftest.py conftest.py
+
 # Test fixtures: the committed snapshot baseline that the snapshot tests
 # (verify()) compare generated output against. Without it the tests find
 # the expected files absent and write them fresh every run instead of

regenerate_snapshots.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Regenerate the .page snapshots under test-data/ from the current sources.
+#
+# Run from the top level of the checkout to regenerate -- the main checkout or
+# any worktree:
+#
+#     ./regenerate_snapshots.sh
+#
+# The suite runs with --regenerate-snapshots, which makes a content test whose
+# rendered output differs rewrite its .page file instead of failing; see
+# ContentTester.verify in sparkmeter/web/unittestutils.py. Matching snapshots
+# are not touched, so the diff is the set of pages the source change affected.
+# Orphans are not pruned: the .page file of a renamed or deleted test is left
+# in place and is not reported, so finding those is a manual step.
+#
+set -euo pipefail
+
+# The image build derives the version from .git via hatch-vcs. In a worktree
+# .git is a file pointing elsewhere rather than a directory, so that fails and
+# SETUPTOOLS_SCM_PRETEND_VERSION supplies the version instead. Built as its own
+# step because `docker compose run --build` cannot forward --build-arg.
+docker compose -f docker-compose.test.yml build \
+    --build-arg SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+g$(git rev-parse --short HEAD)" \
+    test
+
+# Compose namespaces its own containers by project, but this one is created
+# with an explicit --name, so it needs the checkout in its name too. Otherwise
+# the pre-run `docker rm -f` below would remove a container another checkout is
+# using: docker-compose.test.yml is written so two checkouts can run their
+# suites concurrently.
+container="$(basename "$PWD")-snapshot-regen"
+
+docker rm -f "$container" >/dev/null 2>&1 || true
+
+# The run's status is captured rather than allowed to abort the script: a test
+# failing for a reason unrelated to snapshots would otherwise end the run
+# before the copy back, discarding every snapshot it had already rewritten.
+status=0
+docker compose -f docker-compose.test.yml run --name "$container" test \
+    uv run pytest -n auto --regenerate-snapshots || status=$?
+
+# The test image COPYs the source in rather than bind-mounting the working
+# tree, so the rewritten files exist only inside the container -- hence the
+# name instead of --rm, and this copy back.
+if ! docker cp "$container:/app/test-data/." ./test-data/; then
+    echo "Failed to copy the regenerated snapshots out of the container." >&2
+    echo "Leaving the container in place so the run can still be retrieved:" >&2
+    echo "  docker cp $container:/app/test-data/. ./test-data/" >&2
+    exit 1
+fi
+
+docker rm -f "$container" >/dev/null || true
+
+exit "$status"

scripts/run_coverage.sh

@@ -24,9 +24,30 @@ docker compose -f docker-compose.test.yml build \
 # pytest-coverage.txt for the PR report comment. All three are produced in the
 # container, tarred to stdout, and unpacked on the host. pytest's output is teed
 # to stderr, not stdout, so stdout carries only the tar stream.
+#
+# pytest's exit status has to be carried out by hand. `pipefail` makes the
+# pipeline report pytest's status rather than tee's, it is captured before tar
+# runs, and the container exits with it. Otherwise the container's status is
+# tar's: an ordinary red suite still failed the job, but only because
+# --no-cov-on-fail suppresses .coverage.xml and tar then errors on the missing
+# file. A run that writes the reports and still exits non-zero -- pytest's
+# exit 5, "no tests collected", from a -k that matches nothing -- left tar
+# succeeding and the job green. On the host the status is captured rather than
+# allowed to abort, so the script runs to the end and reports it there instead
+# of dying mid-script; when coverage did reach the host, the diff-cover report
+# below is still produced.
+#
+# The artifacts are cleared first so that a run which fails before writing them
+# cannot leave the previous run's files behind to be read as current -- the gate
+# below reads .coverage.xml, and the CI report steps read the other two.
+rm -f .coverage.xml junit.xml pytest-coverage.txt
+
+set +e
 docker compose -f docker-compose.test.yml run --rm test \
-    sh -c 'uv run pytest -n auto --junitxml=/app/junit.xml 2>&1 | tee /app/pytest-coverage.txt 1>&2; tar -c -C /app .coverage.xml junit.xml pytest-coverage.txt' \
+    sh -c 'set -o pipefail; uv run pytest -n auto --junitxml=/app/junit.xml 2>&1 | tee /app/pytest-coverage.txt 1>&2; status=$?; tar -c -C /app .coverage.xml junit.xml pytest-coverage.txt; exit $status' \
     | tar -x
+tests_rc=${PIPESTATUS[0]}
+set -e
 
 # diff-cover gates the changed lines and always prints a terminal report. It
 # also writes a markdown report, which is appended to the GitHub Actions job
@@ -40,16 +61,35 @@ fi
 
 # --fail-under exits non-zero when changed-line coverage is below the threshold.
 # Capture that instead of aborting so the report still reaches the job summary.
-set +e
-uvx "diff-cover==10.3.0" .coverage.xml \
-    --compare-branch "$compare_branch" \
-    --fail-under "$fail_under" \
-    --format "$formats"
-diff_cover_rc=$?
-set -e
+#
+# A suite that fails writes no coverage at all (--no-cov-on-fail in setup.cfg),
+# so there is nothing to gate. Skip it and say so, rather than letting
+# diff-cover fail on the missing file with a traceback that reads like a broken
+# tool instead of a failing test. The suite's own status is reported below.
+diff_cover_rc=0
+if [ -f .coverage.xml ]; then
+    set +e
+    uvx "diff-cover==10.3.0" .coverage.xml \
+        --compare-branch "$compare_branch" \
+        --fail-under "$fail_under" \
+        --format "$formats"
+    diff_cover_rc=$?
+    set -e
+else
+    # Drop any report from a previous run so the summary below cannot append a
+    # stale one for a gate that did not run.
+    rm -f "$report_md"
+    echo "No coverage was produced, so the coverage gate is skipped." >&2
+fi
 
 if [ -n "${GITHUB_STEP_SUMMARY:-}" ] && [ -f "$report_md" ]; then
     cat "$report_md" >> "$GITHUB_STEP_SUMMARY"
 fi
 
+# A failing suite fails the job. It is reported ahead of the coverage gate
+# because a red run makes the coverage number meaningless.
+if [ "$tests_rc" -ne 0 ]; then
+    exit "$tests_rc"
+fi
+
 exit "$diff_cover_rc"