From fb88daec9c70f31180b818ae7ce5ed95f10b30e4 Mon Sep 17 00:00:00 2001 From: hasansezertasan Date: Wed, 19 Aug 2026 18:02:44 +0300 Subject: [PATCH] feat: treat documentation examples as real, tested code (include_docs) Documentation snippets now live as importable modules under `docs/examples/` and are pulled into the docs with `literalinclude`, so every prose example is type-checked, linted, and imported by the test suite instead of drifting silently. - add `docs/examples/version_lookup.py` and reference it from `usage.rst` - add `tests/test_docs_examples.py` (importability + behaviour coverage) - widen ruff/mypy/basedpyright/ty/pyrefly/pylint/zuban scope to `docs/examples` - add a `docs-doctest` tox env (sphinx doctest builder) to `env_list` and a matching `mise` task, plus `sphinx.ext.doctest` - document the rationale in ADR-028 and refresh golden files + render tests --- docs/adr/028-tested-documentation-examples.md | 29 ++++++++++++ ...if include_docs %}conf.py{% endif %}.jinja | 1 + .../version_lookup.py.jinja | 14 ++++++ ... include_docs %}usage.rst{% endif %}.jinja | 15 +++++-- template/mise.toml.jinja | 4 ++ template/pyproject.toml.jinja | 38 +++++++++++++--- ...s %}test_docs_examples.py{% endif %}.jinja | 39 ++++++++++++++++ tests/test_golden_files/pyproject_full.toml | 32 ++++++++++--- .../test_golden_files/pyproject_library.toml | 32 ++++++++++--- tests/test_render_validity.py | 45 +++++++++++++++++++ 10 files changed, 228 insertions(+), 21 deletions(-) create mode 100644 docs/adr/028-tested-documentation-examples.md create mode 100644 template/docs/{% if include_docs %}examples{% endif %}/version_lookup.py.jinja create mode 100644 template/tests/{% if include_docs %}test_docs_examples.py{% endif %}.jinja diff --git a/docs/adr/028-tested-documentation-examples.md b/docs/adr/028-tested-documentation-examples.md new file mode 100644 index 0000000..6493204 --- /dev/null +++ b/docs/adr/028-tested-documentation-examples.md @@ -0,0 +1,29 @@ +# ADR-028: Tested documentation examples + +## Context + +Pasted Python fences in generated documentation can silently drift from the +package API: neither the test suite nor the type-checker and lint gates execute +them. The template's high coverage threshold and broad static-analysis matrix +otherwise make that inconsistency particularly easy to miss. + +## Decision + +Generated projects keep complete Python examples in `docs/examples/`. Sphinx +pages render these files with `literalinclude`, so published code is the exact +source that the regular pytest suite imports and, when appropriate, calls. + +`docs/examples/` is explicitly included in Ruff, mypy, basedpyright, ty, +pyrefly, zuban, and pylint scope. It is not part of the package or coverage +source set: examples document the package rather than constitute product code. + +The Sphinx `doctest` extension is enabled, with a `docs-doctest` tox environment +included in the default CI run for inline `>>>` snippets that cannot use +`literalinclude`. + +## Consequences + +Documentation code now fails the same local and CI checks as a stale import in +the application. Examples remain close to the docs and out of built wheels; +users who need distributable demonstrations can use the independent +`include_examples` scaffold option. diff --git a/template/docs/{% if include_docs %}conf.py{% endif %}.jinja b/template/docs/{% if include_docs %}conf.py{% endif %}.jinja index 8c2353f..d6f0dc0 100644 --- a/template/docs/{% if include_docs %}conf.py{% endif %}.jinja +++ b/template/docs/{% if include_docs %}conf.py{% endif %}.jinja @@ -28,6 +28,7 @@ copyright = f"{_build_date:%Y}, {{ author_full_name }}" # noqa: A001 # -- General configuration --------------------------------------------------- extensions = [ "sphinx.ext.autodoc", + "sphinx.ext.doctest", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.autosectionlabel", diff --git a/template/docs/{% if include_docs %}examples{% endif %}/version_lookup.py.jinja b/template/docs/{% if include_docs %}examples{% endif %}/version_lookup.py.jinja new file mode 100644 index 0000000..81b8766 --- /dev/null +++ b/template/docs/{% if include_docs %}examples{% endif %}/version_lookup.py.jinja @@ -0,0 +1,14 @@ +"""Look up the version of the installed distribution.""" + +from importlib.metadata import version + +from {{github_repo_name}}.__metadata__ import PROJECT_NAME + + +def version_lookup() -> str: + """Return the installed distribution version for this project. + + Returns: + str: The installed distribution version. + """ + return version(PROJECT_NAME) diff --git a/template/docs/{% if include_docs %}usage.rst{% endif %}.jinja b/template/docs/{% if include_docs %}usage.rst{% endif %}.jinja index b977f95..8baa705 100644 --- a/template/docs/{% if include_docs %}usage.rst{% endif %}.jinja +++ b/template/docs/{% if include_docs %}usage.rst{% endif %}.jinja @@ -4,11 +4,20 @@ Usage As a library ------------ -To use ``{{github_repo_name}}`` in a project: +Look up the installed distribution version: -.. code-block:: python +.. literalinclude:: examples/version_lookup.py + :language: python + :caption: examples/version_lookup.py - import {{github_repo_name}} +For short interactive snippets embedded in prose, the ``docs-doctest`` task +executes ``>>>`` blocks too: + +.. doctest:: + + >>> from {{github_repo_name}}.__metadata__ import PROJECT_NAME + >>> PROJECT_NAME + '{{github_repo_name}}' {%- if include_cli %} As a command-line tool diff --git a/template/mise.toml.jinja b/template/mise.toml.jinja index 0f4e45f..aa33e77 100644 --- a/template/mise.toml.jinja +++ b/template/mise.toml.jinja @@ -60,6 +60,10 @@ run = "uv run --locked tox run -e docs-build" [tasks.docs-serve] description = "Serve documentation locally" run = "uv run --locked tox run -e docs-server" + +[tasks.docs-doctest] +description = "Run documentation doctests" +run = "uv run --locked tox run -e docs-doctest" {%- endif %} {% raw %} diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 6acf42c..7b39bb8 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -517,7 +517,7 @@ fix = true output-format = "full" preview = true show-fixes = true -src = ["src", "tests"{% if include_examples %}, "examples"{% endif %}{% if include_worker %}, "scripts"{% endif %}] +src = ["src", "tests"{% if include_examples %}, "examples"{% endif %}{% if include_docs %}, "docs/examples"{% endif %}{% if include_worker %}, "scripts"{% endif %}] target-version = "py310" unsafe-fixes = true @@ -590,6 +590,9 @@ convention = "pep257" "examples/**/*.py" = ["INP001"] {%- endif %} {%- if include_docs %} +# Documentation examples are importable modules tested by +# ``tests/test_docs_examples.py``; they are not application packages. +"docs/examples/**/*.py" = ["INP001"] # The Sphinx config is a standalone module, not part of an importable package. "docs/conf.py" = ["INP001"] # The docs warning-allowlist gate is a standalone script (not an importable @@ -621,7 +624,7 @@ parenthesize-tuple-in-subscript = true [tool.mypy] -files = ["src"] +files = ["src"{% if include_docs %}, "docs/examples"{% endif %}] pretty = true python_version = "3.10" strict = true @@ -629,7 +632,7 @@ strict = true [tool.basedpyright] exclude = [".venv"] -include = ["src/{{github_repo_name}}"] +include = ["src/{{github_repo_name}}"{% if include_docs %}, "docs/examples"{% endif %}] pythonVersion = "3.10" venv = ".venv" # venvPath is the directory *containing* the venv, not the venv itself; ".venv" @@ -649,7 +652,7 @@ reportImplicitStringConcatenation = "none" [tool.ty.src] -include = ["src", "tests"] +include = ["src", "tests"{% if include_docs %}, "docs/examples"{% endif %}] respect-ignore-files = false @@ -663,11 +666,11 @@ error-on-warning = true [tool.pyrefly] python-version = "3.10" -project-includes = ["src"] +project-includes = ["src"{% if include_docs %}, "docs/examples"{% endif %}] [tool.pylint.main] # pylint runs as an always-on gate that deliberately overlaps ruff's PL* rules. -# Its canonical invocation is `pylint src`, so it never descends into tests; this +# Its canonical invocation is `pylint src{% if include_docs %} docs/examples{% endif %}`, so it never descends into tests; this # ignore-paths is a defensive scope guard for broader invocations (`pylint .`, # filename-passing hooks). Tests are linted by ruff, not pylint. ignore-paths = ["^tests/.*$"] @@ -720,6 +723,11 @@ datas = "datas" # ``tox -e worker`` when a broker is up (e.g. inside the devcontainer). env_list = [ "style", +{%- if include_docs %} + # Inline ``>>>`` snippets are part of the tested-documentation contract, + # so run them in the default CI ``tox run`` invocation. + "docs-doctest", +{%- endif %} {%- if include_cli %} "cli", {%- endif %} @@ -808,10 +816,16 @@ commands = [ "zuban", "check", "src", +{%- if include_docs %} + "docs/examples", +{%- endif %} ], [ "pylint", "src", +{%- if include_docs %} + "docs/examples", +{%- endif %} ], [ "python", @@ -913,6 +927,18 @@ runner = "uv-venv-runner" set_env = { PYTHONUNBUFFERED = "1" } +[tool.tox.env.docs-doctest] +# ``literalinclude`` keeps complete examples in tested Python modules; this +# separate builder executes the occasional ``>>>`` snippet that belongs inline +# with prose. +commands = [["sphinx-build", "-b", "doctest", "docs", "docs/_build/doctest"]] +extras = ["all"] +dependency_groups = ["docs"] +description = "Run documentation doctests" +runner = "uv-venv-runner" +set_env = { PYTHONUNBUFFERED = "1" } + + [tool.tox.env.docs-linkcheck] # On-demand link checker. Also run weekly (non-blocking) by docs-linkcheck.yml. # Hits the network, so it is deliberately not part of the `style` env or PR CI. diff --git a/template/tests/{% if include_docs %}test_docs_examples.py{% endif %}.jinja b/template/tests/{% if include_docs %}test_docs_examples.py{% endif %}.jinja new file mode 100644 index 0000000..799d0b4 --- /dev/null +++ b/template/tests/{% if include_docs %}test_docs_examples.py{% endif %}.jinja @@ -0,0 +1,39 @@ +"""Keep the Python modules embedded in the documentation executable.""" + +from __future__ import annotations + +import importlib.util +from importlib.metadata import version +from pathlib import Path +from typing import TYPE_CHECKING + +from {{github_repo_name}}.__metadata__ import PROJECT_NAME + +if TYPE_CHECKING: + from types import ModuleType + +EXAMPLES_DIR = Path(__file__).parents[1] / "docs" / "examples" + + +def _load_example(path: Path) -> ModuleType: + """Import one documentation example directly from its source path.""" + spec = importlib.util.spec_from_file_location(f"docs_example_{path.stem}", path) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_all_documentation_examples_are_importable() -> None: + """Every nested Python module in ``docs/examples`` imports successfully.""" + examples = sorted(EXAMPLES_DIR.rglob("*.py")) + assert examples, "docs/examples must contain at least one tested module" + for path in examples: + _load_example(path) + + +def test_version_lookup_example_uses_the_installed_distribution() -> None: + """The usage-page example resolves the same version as package metadata.""" + example = _load_example(EXAMPLES_DIR / "version_lookup.py") + assert example.version_lookup() == version(PROJECT_NAME) diff --git a/tests/test_golden_files/pyproject_full.toml b/tests/test_golden_files/pyproject_full.toml index 036e3ec..2f1626d 100644 --- a/tests/test_golden_files/pyproject_full.toml +++ b/tests/test_golden_files/pyproject_full.toml @@ -390,7 +390,7 @@ fix = true output-format = "full" preview = true show-fixes = true -src = ["src", "tests", "examples", "scripts"] +src = ["src", "tests", "examples", "docs/examples", "scripts"] target-version = "py310" unsafe-fixes = true @@ -460,6 +460,9 @@ convention = "pep257" # don't need return-value documentation on their fixtures and helpers. "tests/**/*.py" = ["S101", "PLR2004", "PLC2701", "DOC201"] "examples/**/*.py" = ["INP001"] +# Documentation examples are importable modules tested by +# ``tests/test_docs_examples.py``; they are not application packages. +"docs/examples/**/*.py" = ["INP001"] # The Sphinx config is a standalone module, not part of an importable package. "docs/conf.py" = ["INP001"] # The docs warning-allowlist gate is a standalone script (not an importable @@ -486,7 +489,7 @@ parenthesize-tuple-in-subscript = true [tool.mypy] -files = ["src"] +files = ["src", "docs/examples"] pretty = true python_version = "3.10" strict = true @@ -494,7 +497,7 @@ strict = true [tool.basedpyright] exclude = [".venv"] -include = ["src/example"] +include = ["src/example", "docs/examples"] pythonVersion = "3.10" venv = ".venv" # venvPath is the directory *containing* the venv, not the venv itself; ".venv" @@ -512,7 +515,7 @@ reportImplicitStringConcatenation = "none" [tool.ty.src] -include = ["src", "tests"] +include = ["src", "tests", "docs/examples"] respect-ignore-files = false @@ -526,11 +529,11 @@ error-on-warning = true [tool.pyrefly] python-version = "3.10" -project-includes = ["src"] +project-includes = ["src", "docs/examples"] [tool.pylint.main] # pylint runs as an always-on gate that deliberately overlaps ruff's PL* rules. -# Its canonical invocation is `pylint src`, so it never descends into tests; this +# Its canonical invocation is `pylint src docs/examples`, so it never descends into tests; this # ignore-paths is a defensive scope guard for broader invocations (`pylint .`, # filename-passing hooks). Tests are linted by ruff, not pylint. ignore-paths = ["^tests/.*$"] @@ -583,6 +586,9 @@ datas = "datas" # ``tox -e worker`` when a broker is up (e.g. inside the devcontainer). env_list = [ "style", + # Inline ``>>>`` snippets are part of the tested-documentation contract, + # so run them in the default CI ``tox run`` invocation. + "docs-doctest", "cli", "3.14", "3.13", @@ -667,10 +673,12 @@ commands = [ "zuban", "check", "src", + "docs/examples", ], [ "pylint", "src", + "docs/examples", ], [ "python", @@ -770,6 +778,18 @@ runner = "uv-venv-runner" set_env = { PYTHONUNBUFFERED = "1" } +[tool.tox.env.docs-doctest] +# ``literalinclude`` keeps complete examples in tested Python modules; this +# separate builder executes the occasional ``>>>`` snippet that belongs inline +# with prose. +commands = [["sphinx-build", "-b", "doctest", "docs", "docs/_build/doctest"]] +extras = ["all"] +dependency_groups = ["docs"] +description = "Run documentation doctests" +runner = "uv-venv-runner" +set_env = { PYTHONUNBUFFERED = "1" } + + [tool.tox.env.docs-linkcheck] # On-demand link checker. Also run weekly (non-blocking) by docs-linkcheck.yml. # Hits the network, so it is deliberately not part of the `style` env or PR CI. diff --git a/tests/test_golden_files/pyproject_library.toml b/tests/test_golden_files/pyproject_library.toml index 2711cb3..01e6534 100644 --- a/tests/test_golden_files/pyproject_library.toml +++ b/tests/test_golden_files/pyproject_library.toml @@ -297,7 +297,7 @@ fix = true output-format = "full" preview = true show-fixes = true -src = ["src", "tests", "examples"] +src = ["src", "tests", "examples", "docs/examples"] target-version = "py310" unsafe-fixes = true @@ -358,6 +358,9 @@ convention = "pep257" # don't need return-value documentation on their fixtures and helpers. "tests/**/*.py" = ["S101", "PLR2004", "PLC2701", "DOC201"] "examples/**/*.py" = ["INP001"] +# Documentation examples are importable modules tested by +# ``tests/test_docs_examples.py``; they are not application packages. +"docs/examples/**/*.py" = ["INP001"] # The Sphinx config is a standalone module, not part of an importable package. "docs/conf.py" = ["INP001"] # The docs warning-allowlist gate is a standalone script (not an importable @@ -377,7 +380,7 @@ parenthesize-tuple-in-subscript = true [tool.mypy] -files = ["src"] +files = ["src", "docs/examples"] pretty = true python_version = "3.10" strict = true @@ -385,7 +388,7 @@ strict = true [tool.basedpyright] exclude = [".venv"] -include = ["src/example"] +include = ["src/example", "docs/examples"] pythonVersion = "3.10" venv = ".venv" # venvPath is the directory *containing* the venv, not the venv itself; ".venv" @@ -395,7 +398,7 @@ venvPath = "." [tool.ty.src] -include = ["src", "tests"] +include = ["src", "tests", "docs/examples"] respect-ignore-files = false @@ -409,11 +412,11 @@ error-on-warning = true [tool.pyrefly] python-version = "3.10" -project-includes = ["src"] +project-includes = ["src", "docs/examples"] [tool.pylint.main] # pylint runs as an always-on gate that deliberately overlaps ruff's PL* rules. -# Its canonical invocation is `pylint src`, so it never descends into tests; this +# Its canonical invocation is `pylint src docs/examples`, so it never descends into tests; this # ignore-paths is a defensive scope guard for broader invocations (`pylint .`, # filename-passing hooks). Tests are linted by ruff, not pylint. ignore-paths = ["^tests/.*$"] @@ -460,6 +463,9 @@ max-branches = 20 # ``tox -e worker`` when a broker is up (e.g. inside the devcontainer). env_list = [ "style", + # Inline ``>>>`` snippets are part of the tested-documentation contract, + # so run them in the default CI ``tox run`` invocation. + "docs-doctest", "3.14", "3.13", "3.12", @@ -539,10 +545,12 @@ commands = [ "zuban", "check", "src", + "docs/examples", ], [ "pylint", "src", + "docs/examples", ], [ "python", @@ -642,6 +650,18 @@ runner = "uv-venv-runner" set_env = { PYTHONUNBUFFERED = "1" } +[tool.tox.env.docs-doctest] +# ``literalinclude`` keeps complete examples in tested Python modules; this +# separate builder executes the occasional ``>>>`` snippet that belongs inline +# with prose. +commands = [["sphinx-build", "-b", "doctest", "docs", "docs/_build/doctest"]] +extras = ["all"] +dependency_groups = ["docs"] +description = "Run documentation doctests" +runner = "uv-venv-runner" +set_env = { PYTHONUNBUFFERED = "1" } + + [tool.tox.env.docs-linkcheck] # On-demand link checker. Also run weekly (non-blocking) by docs-linkcheck.yml. # Hits the network, so it is deliberately not part of the `style` env or PR CI. diff --git a/tests/test_render_validity.py b/tests/test_render_validity.py index 8a78973..f86a939 100644 --- a/tests/test_render_validity.py +++ b/tests/test_render_validity.py @@ -297,3 +297,48 @@ def test_docs_on_by_default_keeps_sphinx_subsystem( pyproject["project"]["urls"]["documentation"] == "https://octocat.github.io/example" ) + + +def test_documentation_examples_are_rendered_and_checked( + render: Callable[..., Path], +) -> None: + """Docs examples are literal-included modules covered by every quality gate.""" + root = render() + example = root / "docs" / "examples" / "version_lookup.py" + assert example.is_file() + assert "def version_lookup()" in example.read_text(encoding="utf-8") + + usage = (root / "docs" / "usage.rst").read_text(encoding="utf-8") + assert ".. literalinclude:: examples/version_lookup.py" in usage + + test = (root / "tests" / "test_docs_examples.py").read_text(encoding="utf-8") + assert "test_all_documentation_examples_are_importable" in test + assert "test_version_lookup_example_uses_the_installed_distribution" in test + assert 'EXAMPLES_DIR.rglob("*.py")' in test + + pyproject = (root / "pyproject.toml").read_text(encoding="utf-8") + for checker_scope in ( + 'files = ["src", "docs/examples"]', + 'include = ["src/example", "docs/examples"]', + 'include = ["src", "tests", "docs/examples"]', + 'project-includes = ["src", "docs/examples"]', + '"docs/examples",', + ): + assert checker_scope in pyproject + assert '[tool.tox.env.docs-doctest]' in pyproject + assert '"docs-doctest",' in pyproject + assert '"sphinx.ext.doctest",' in (root / "docs" / "conf.py").read_text( + encoding="utf-8" + ) + + +def test_docs_off_omits_documentation_examples_and_checks( + render: Callable[..., Path], +) -> None: + """The docs toggle owns the complete tested-examples subsystem.""" + root = render(include_docs=False) + assert not (root / "docs" / "examples").exists() + assert not (root / "tests" / "test_docs_examples.py").exists() + assert '"docs/examples",' not in (root / "pyproject.toml").read_text( + encoding="utf-8" + )