diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f0cbdd..565e7dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,13 +12,18 @@ jobs: matrix: python-version: ["3.12", "3.13"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + # v6/v5 targeted Node 20, which runners now force onto Node 24 with a + # deprecation warning; v6 is the Node-24-native pair. + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - run: pip install -e ".[dev]" - - run: ruff check src tests + - run: ruff check src tests scripts - run: mypy # The default suite is fully offline: a conftest socket guard fails any test # that reaches for a non-loopback address; playwright/live are deselected. - run: python -m pytest -q + # Prose-vs-code drift has shipped twice (PR #1, PR #4) because nothing + # compared the README/CHANGELOG claims to the manifests and the suite. + - run: python scripts/check_consistency.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d05ade..f3d9457 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,10 +27,15 @@ Requires Python >= 3.12. Optional, for the browser-render path and its tests: Every change must pass the same gate CI runs. Run it before you push: ```bash -.venv/bin/ruff check src tests && .venv/bin/mypy && .venv/bin/python -m pytest +.venv/bin/ruff check src tests scripts && .venv/bin/mypy && \ + .venv/bin/python -m pytest && .venv/bin/python scripts/check_consistency.py ``` - **`ruff check`** — lint. **`mypy`** — strict type-checking. **`pytest`** — the offline suite. +- **`check_consistency.py`** — asserts the repo's claims about itself are true: the version in + `pyproject.toml`, `__init__.py` and a released `CHANGELOG.md` section all agree, and every + "N tests" figure in the README equals what pytest actually collects. If you add tests or + bump the version, this is what tells you which prose went stale. - The default suite runs **fully offline**: a `conftest.py` socket guard fails any test that reaches a non-loopback address, so tests stay deterministic. HTTP is faked with `httpx.MockTransport` and fixture HTML. diff --git a/README.md b/README.md index c164e30..7b8d213 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ no telemetry. > *tearsheet (n.): a page torn from a publication and filed as proof it ran.* -**Trust status:** qualified for heavy usage 2026-07-16 — 243 tests, a falsifiable live +**Trust status:** qualified for heavy usage 2026-07-16 — 247 tests, a falsifiable live eval harness (verdict GREEN), and zero fabrications across the tool's entire recorded history. Its documented failure mode is *omission*, and the guards exist to make every omission loud. See [Trust](#trust). @@ -161,7 +161,7 @@ a research tool for reading the public web — not for evading paywalls or bot d "Can it be trusted for heavy usage?" is a measurement here, not a feeling. -- **Offline suite (243 tests, runs in the gate)**: guard boundary pins, cache-poisoning +- **Offline suite (247 tests, runs in the gate)**: guard boundary pins, cache-poisoning regressions, truncation honesty, charset torture, structure torture, adversarial robustness — enforced fully offline by a loopback-only socket guard. The five REAL pages that defined the tool's probation (quo, smith.ai, dialpad, heyrosie, a LinkedIn @@ -195,7 +195,7 @@ for figures you'll quote, treat a suspiciously small extraction of a rich page a .venv/bin/ruff check src tests && .venv/bin/mypy && .venv/bin/python -m pytest ``` -TDD throughout; the default suite (243 tests) runs entirely offline — `httpx.MockTransport`, +TDD throughout; the default suite (247 tests) runs entirely offline — `httpx.MockTransport`, fixture HTML, and a conftest socket guard that fails any test reaching for a non-loopback address. Extras: `pytest -m playwright` (real chromium, local server), `pytest -m live` (real network). The live trust evaluation lives in `evals/` (see [Trust](#trust)). diff --git a/scripts/check_consistency.py b/scripts/check_consistency.py new file mode 100644 index 0000000..b13ac8c --- /dev/null +++ b/scripts/check_consistency.py @@ -0,0 +1,123 @@ +"""Assert the repo's claims about itself are true. + +Two classes of drift have shipped here twice, both caught only by luck: + + * PR #1 — "Fix version and test-count inconsistencies" + * PR #4 — README said "Since v0.1.5" while both manifests still said 0.1.4 + and the CHANGELOG carried the change under [Unreleased] + +Neither the gate nor CI could see either one, because nothing compared the +prose to the code. This script does. Run it in the gate. + +Checks, all against `pyproject.toml`'s version as canonical: + 1. `src/tearsheet/__init__.py.__version__` matches it. + 2. `CHANGELOG.md` has a released `## [X.Y.Z]` section for it — this is the + one that catches "documented as shipped while still sitting in + [Unreleased]". + 3. Every "N tests" claim in `README.md` equals the number pytest actually + collects for the default (offline) suite. + +Exit 0 when everything agrees; 1 with a list of mismatches otherwise. +""" + +from __future__ import annotations + +import re +import subprocess +import sys +import tomllib +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# "243 tests", "(243 tests, runs in the gate)", "suite (243 tests)" — any prose +# claim about how many tests exist. Deliberately broad: a claim this script +# cannot see is a claim that can rot. +_README_TEST_CLAIM = re.compile(r"\(?(\d{2,4})\s+tests\b") +_COLLECTED = re.compile(r"(\d+)(?:/\d+)?\s+tests? collected") + + +def canonical_version() -> str: + data = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) + version = data["project"]["version"] + if not isinstance(version, str): # pragma: no cover - malformed manifest + raise TypeError("pyproject.toml [project].version must be a string") + return version + + +def check_dunder_version(expected: str) -> list[str]: + path = REPO_ROOT / "src" / "tearsheet" / "__init__.py" + m = re.search(r'__version__\s*=\s*"([^"]+)"', path.read_text()) + if not m: + return [f"{path.name}: no `__version__ = \"X.Y.Z\"` found"] + if m.group(1) != expected: + return [f"{path.name}: __version__ {m.group(1)!r} != {expected!r}"] + return [] + + +def check_changelog_released(expected: str) -> list[str]: + path = REPO_ROOT / "CHANGELOG.md" + text = path.read_text() + if re.search(rf"^## \[{re.escape(expected)}\]", text, re.M): + return [] + return [ + f"{path.name}: no released `## [{expected}]` section — the current version is " + "still unreleased, or the section was never added" + ] + + +def collected_test_count() -> tuple[int | None, list[str]]: + """How many tests the DEFAULT suite collects (what the gate and README mean).""" + proc = subprocess.run( # noqa: S603 - fixed argv, no shell + [sys.executable, "-m", "pytest", "--collect-only", "-q"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + m = None + for line in reversed(proc.stdout.splitlines()): + m = _COLLECTED.search(line) + if m: + break + if not m: + return None, ["could not determine collected test count from pytest output"] + return int(m.group(1)), [] + + +def check_readme_test_counts(actual: int) -> list[str]: + path = REPO_ROOT / "README.md" + problems = [] + for lineno, line in enumerate(path.read_text().splitlines(), start=1): + for m in _README_TEST_CLAIM.finditer(line): + claimed = int(m.group(1)) + if claimed != actual: + problems.append( + f"{path.name}:{lineno}: claims {claimed} tests, pytest collects {actual}" + ) + return problems + + +def main() -> int: + expected = canonical_version() + problems: list[str] = [] + problems.extend(check_dunder_version(expected)) + problems.extend(check_changelog_released(expected)) + + actual, count_problems = collected_test_count() + problems.extend(count_problems) + if actual is not None: + problems.extend(check_readme_test_counts(actual)) + + if problems: + print(f"Consistency check FAILED. pyproject.toml version is {expected!r}.") + for p in problems: + print(f" - {p}") + return 1 + + print(f"Consistency OK. Version {expected} everywhere; README matches {actual} tests.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())