From 9e4eebd89d3e6c4ab27194bc133c314cd01cad08 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Sun, 2 Aug 2026 19:08:01 +1000 Subject: [PATCH 1/4] CI: Run docstring examples via pytest --doctest-modules Wires the docstring-example sweep from gh-866 into ci.yml so examples cannot silently rot again after the gh-864 fixes. A new quantecon/conftest.py resolves the two blockers raised in gh-866: - An autouse fixture restores NumPy print options after each test, so the game_theory examples that set precision=4 no longer leak process-global state into doctests collected later (previously 8 spurious failures in markov, optimize and random). - collect_ignore excludes util/notebooks.py (fetches over the network) and util/timing.py (prints wall-clock durations), keeping the rendered docs free of `# doctest: +SKIP` directives. The CI step runs on Linux only: one platform is enough to stop drift, and it sidesteps --ignore-glob path-separator issues on Windows. The DeprecationWarning from util/array.py::searchsorted needs no handling since no strict warning filter is configured. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 7 +++++++ quantecon/conftest.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 quantecon/conftest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3669c470..6abc089bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,13 @@ jobs: shell: bash -l {0} run: | pytest quantecon + - name: Run docstring examples (doctest) + # one platform is enough to stop examples drifting, and keeping + # this off Windows avoids path-separator issues with --ignore-glob + if: runner.os == 'Linux' + shell: bash -l {0} + run: | + pytest --doctest-modules quantecon --ignore-glob='*/tests/*' coverage: name: Coverage diff --git a/quantecon/conftest.py b/quantecon/conftest.py new file mode 100644 index 000000000..ab5413be0 --- /dev/null +++ b/quantecon/conftest.py @@ -0,0 +1,33 @@ +""" +Pytest configuration for the ``--doctest-modules`` run (see gh-866). + +""" +import numpy as np +import pytest + +# These modules have docstring examples that cannot pass verbatim: +# `fetch_nb_dependencies` fetches over the network, and the `timing` +# examples print wall-clock durations that vary per run. Exclude them +# here rather than annotating the examples with `# doctest: +SKIP`, +# which would render visibly in the published docs. +collect_ignore = [ + "util/notebooks.py", + "util/timing.py", +] + + +@pytest.fixture(autouse=True) +def _restore_printoptions(): + """ + Restore NumPy print options after each test. + + Several `game_theory` docstring examples call + `np.set_printoptions(precision=4)` for readability and deliberately + do not restore it. Print options are process-global, so without + this fixture every doctest collected after those examples would + render arrays at the leaked precision and fail. + + """ + saved = np.get_printoptions() + yield + np.set_printoptions(**saved) From cc67114e850b4c85a24f429cc1bb9eb745635bc4 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Mon, 3 Aug 2026 09:08:26 +1000 Subject: [PATCH 2/4] CI: move doctest conftest.py to the repository root qe_apidoc.py discovers Tools pages by globbing quantecon/[a-z0-9]*.py, so an in-package conftest.py would be picked up as a public module and fail the docs-drift check; the root location also keeps a module that imports pytest out of the installed wheel. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 6 +++--- quantecon/conftest.py => conftest.py | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) rename quantecon/conftest.py => conftest.py (60%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6abc089bc..1fccd1d4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,14 +50,14 @@ jobs: - name: flake8 Tests shell: bash -l {0} run: | - flake8 --select=F401,F405,E231 quantecon + flake8 --select=F401,F405,E231 quantecon conftest.py - name: Run Tests (pytest) shell: bash -l {0} run: | pytest quantecon - name: Run docstring examples (doctest) - # one platform is enough to stop examples drifting, and keeping - # this off Windows avoids path-separator issues with --ignore-glob + # one platform is enough to stop examples drifting, and float + # reprs are the platform-sensitive part of doctest output if: runner.os == 'Linux' shell: bash -l {0} run: | diff --git a/quantecon/conftest.py b/conftest.py similarity index 60% rename from quantecon/conftest.py rename to conftest.py index ab5413be0..9fb2fcc8f 100644 --- a/quantecon/conftest.py +++ b/conftest.py @@ -1,6 +1,14 @@ """ Pytest configuration for the ``--doctest-modules`` run (see gh-866). +This lives at the repository root rather than inside the package on +purpose. `docs/qe_apidoc.py` discovers the "Tools" pages by globbing +``../quantecon/[a-z0-9]*.py``, so a `quantecon/conftest.py` would be +picked up as a public module and generate a docs page for itself, +failing the docs-drift check in CI. Keeping it here also keeps a +module that imports pytest -- not a runtime dependency -- out of the +installed wheel. + """ import numpy as np import pytest @@ -9,10 +17,11 @@ # `fetch_nb_dependencies` fetches over the network, and the `timing` # examples print wall-clock durations that vary per run. Exclude them # here rather than annotating the examples with `# doctest: +SKIP`, -# which would render visibly in the published docs. +# which would render visibly in the published docs. Paths are relative +# to this file, i.e. to the repository root. collect_ignore = [ - "util/notebooks.py", - "util/timing.py", + "quantecon/util/notebooks.py", + "quantecon/util/timing.py", ] From 1132b9f37e7a00609deb1d7312231149629e9ff6 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Wed, 9 Sep 2026 19:47:05 +1000 Subject: [PATCH 3/4] TEST: run docstring examples without comparing output Replaces the `--doctest-modules` CI step and its root `conftest.py` with a single test that executes every docstring example and checks only that it does not raise. Comparing example output is what made the previous design need machinery. Global NumPy print options leak between modules, causing nine spurious failures, and two modules could not pass verbatim at all. Neither applies once output is not compared: leaked print options are invisible, and `util/timing.py`'s wall-clock durations no longer matter, so only `util/notebooks.py`, which fetches over the network, is excluded -- and that module is deprecated for removal in v1.0. What this gives up is drift in the *text* of an example's output, the category that breaks on every NumPy repr change and would commit us to re-pasting expected output indefinitely. What it keeps is every failure that breaks a reader who copies an example: missing imports, renamed functions, changed signatures. Running as an ordinary test rather than a separate CI step means the sweep covers the whole platform matrix, which execution-only tolerates because it has no platform-sensitive output. A collection guard fails if the sweep ever finds nothing, so it cannot silently become a no-op. Closes #866. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 9 +-- conftest.py | 42 ---------- quantecon/tests/test_docstring_examples.py | 92 ++++++++++++++++++++++ 3 files changed, 93 insertions(+), 50 deletions(-) delete mode 100644 conftest.py create mode 100644 quantecon/tests/test_docstring_examples.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fccd1d4a..f3669c470 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,18 +50,11 @@ jobs: - name: flake8 Tests shell: bash -l {0} run: | - flake8 --select=F401,F405,E231 quantecon conftest.py + flake8 --select=F401,F405,E231 quantecon - name: Run Tests (pytest) shell: bash -l {0} run: | pytest quantecon - - name: Run docstring examples (doctest) - # one platform is enough to stop examples drifting, and float - # reprs are the platform-sensitive part of doctest output - if: runner.os == 'Linux' - shell: bash -l {0} - run: | - pytest --doctest-modules quantecon --ignore-glob='*/tests/*' coverage: name: Coverage diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 9fb2fcc8f..000000000 --- a/conftest.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Pytest configuration for the ``--doctest-modules`` run (see gh-866). - -This lives at the repository root rather than inside the package on -purpose. `docs/qe_apidoc.py` discovers the "Tools" pages by globbing -``../quantecon/[a-z0-9]*.py``, so a `quantecon/conftest.py` would be -picked up as a public module and generate a docs page for itself, -failing the docs-drift check in CI. Keeping it here also keeps a -module that imports pytest -- not a runtime dependency -- out of the -installed wheel. - -""" -import numpy as np -import pytest - -# These modules have docstring examples that cannot pass verbatim: -# `fetch_nb_dependencies` fetches over the network, and the `timing` -# examples print wall-clock durations that vary per run. Exclude them -# here rather than annotating the examples with `# doctest: +SKIP`, -# which would render visibly in the published docs. Paths are relative -# to this file, i.e. to the repository root. -collect_ignore = [ - "quantecon/util/notebooks.py", - "quantecon/util/timing.py", -] - - -@pytest.fixture(autouse=True) -def _restore_printoptions(): - """ - Restore NumPy print options after each test. - - Several `game_theory` docstring examples call - `np.set_printoptions(precision=4)` for readability and deliberately - do not restore it. Print options are process-global, so without - this fixture every doctest collected after those examples would - render arrays at the leaked precision and fail. - - """ - saved = np.get_printoptions() - yield - np.set_printoptions(**saved) diff --git a/quantecon/tests/test_docstring_examples.py b/quantecon/tests/test_docstring_examples.py new file mode 100644 index 000000000..89d2e4f9e --- /dev/null +++ b/quantecon/tests/test_docstring_examples.py @@ -0,0 +1,92 @@ +""" +Execute every docstring example in the package, without comparing output. + +Examples are executable specifications: if one stops running, a reader +who copies it gets an error. This sweep catches exactly that -- missing +imports, renamed functions, changed signatures -- and nothing else. + +It deliberately does not compare an example's printed output with the +text in the docstring. That comparison is what `--doctest-modules` +performs, and it fails whenever NumPy changes an array repr or a float +format, independently of whether the example still works. Most of the +31 examples repaired in gh-864 had rotted that way (pre-NumPy-1.14 +array spacing, NumPy 2 scalar reprs), so keeping the comparison would +commit us to re-pasting expected output after every NumPy release. +See gh-866 for the discussion. + +""" +import contextlib +import doctest +import importlib +import io +import pkgutil + +import numpy as np +import pytest + +import quantecon + + +# `fetch_nb_dependencies` fetches over the network, so its example cannot +# run offline. The module is deprecated and is removed in v1.0 (gh-880), +# which retires this entry with it. +SKIP_MODULES = {'quantecon.util.notebooks'} + + +def _collect_doctests(): + """ + Return every docstring in the package that carries examples. + + """ + collected = [] + for module_info in pkgutil.walk_packages(quantecon.__path__, 'quantecon.'): + name = module_info.name + if '.tests' in name or name in SKIP_MODULES: + continue + module = importlib.import_module(name) + for test in doctest.DocTestFinder().find(module): + if test.examples: + collected.append(test) + return collected + + +DOCTESTS = _collect_doctests() + + +def test_doctests_are_collected(): + """ + Guard against the sweep silently becoming a no-op. + + Every test below is generated from `DOCTESTS`, so an empty + collection would report success while checking nothing -- the same + way these examples rotted in the first place. + + """ + assert len(DOCTESTS) > 40 + + +@pytest.mark.parametrize('test', DOCTESTS, ids=lambda test: test.name) +def test_docstring_example_runs(test): + """ + Execute one docstring's examples, discarding their output. + + The examples of a docstring share a namespace, as they do under + `doctest`, so a later line may use names bound by an earlier one. + + """ + printoptions = np.get_printoptions() + namespace = dict(test.globs) + try: + for example in test.examples: + code = compile(example.source, '<%s>' % test.name, 'single') + try: + with contextlib.redirect_stdout(io.StringIO()): + exec(code, namespace) + except Exception: + # An example documenting a traceback is meant to raise. + if example.exc_msg is None: + raise + finally: + # Several `game_theory` examples call `np.set_printoptions` for + # readability and do not restore it; print options are global. + np.set_printoptions(**printoptions) From 66f2bcf39d2f91c049fbcf94108e339dc0df4f49 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Thu, 10 Sep 2026 09:49:03 +1000 Subject: [PATCH 4/4] TEST: Report a failing docstring example by file, line and source A failing example was reported only by docstring name and exception: the compiled snippet had no registered source, so the traceback showed '???' and ':1' for every failure, above fifteen lines of harness code. Register each example with linecache under a '' filename, as doctest's own runner does, and re-raise with the source file, exact line, position within the docstring and the example's source, chaining the original exception so a failure inside library code keeps its traceback. The execution loop moves into a helper so a self-test can drive it with a synthetic docstring: the reporting path only runs when an example is broken, and a mistake in its line arithmetic would otherwise surface at the worst moment. Co-Authored-By: Claude Fable 5.1 --- quantecon/tests/test_docstring_examples.py | 87 ++++++++++++++++++---- 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/quantecon/tests/test_docstring_examples.py b/quantecon/tests/test_docstring_examples.py index 89d2e4f9e..a7f3d443b 100644 --- a/quantecon/tests/test_docstring_examples.py +++ b/quantecon/tests/test_docstring_examples.py @@ -19,6 +19,8 @@ import doctest import importlib import io +import linecache +import os import pkgutil import numpy as np @@ -65,28 +67,87 @@ def test_doctests_are_collected(): assert len(DOCTESTS) > 40 +def _run_examples(test): + """ + Execute the examples of one docstring, discarding their output. + + The examples share a namespace, as they do under `doctest`, so a + later line may use names bound by an earlier one. An example that + raises is reported by source file, line and position, with the + original exception chained so a failure inside library code keeps + its full traceback. + + """ + __tracebackhide__ = True + namespace = dict(test.globs) + for i, example in enumerate(test.examples, 1): + filename = '' % (test.name, i) + # Register the source so a traceback shows the example's line + # instead of '???'; doctest's own runner does the same. + linecache.cache[filename] = ( + len(example.source), None, + example.source.splitlines(keepends=True), filename) + code = compile(example.source, filename, 'single') + try: + with contextlib.redirect_stdout(io.StringIO()): + exec(code, namespace) + except Exception as exc: + # An example documenting a traceback is meant to raise. + if example.exc_msg is not None: + continue + # Both line numbers are zero-based: the docstring's within + # the file, and the example's within the docstring. + lineno = test.lineno + example.lineno + 1 + raise AssertionError( + '%s:%d: example %d of %d in %s raised %s: %s\n' + ' >>> %s' + % (os.path.relpath(test.filename), lineno, i, + len(test.examples), test.name, + type(exc).__name__, exc, + example.source.strip().replace('\n', '\n ... ')) + ) from exc + + @pytest.mark.parametrize('test', DOCTESTS, ids=lambda test: test.name) def test_docstring_example_runs(test): """ Execute one docstring's examples, discarding their output. - The examples of a docstring share a namespace, as they do under - `doctest`, so a later line may use names bound by an earlier one. - """ + __tracebackhide__ = True printoptions = np.get_printoptions() - namespace = dict(test.globs) try: - for example in test.examples: - code = compile(example.source, '<%s>' % test.name, 'single') - try: - with contextlib.redirect_stdout(io.StringIO()): - exec(code, namespace) - except Exception: - # An example documenting a traceback is meant to raise. - if example.exc_msg is None: - raise + _run_examples(test) finally: # Several `game_theory` examples call `np.set_printoptions` for # readability and do not restore it; print options are global. np.set_printoptions(**printoptions) + + +def test_failure_report_names_the_example(): + """ + A broken example is reported by file, line, position and source. + + This path only runs when an example is broken, so exercise it with + a synthetic docstring rather than wait for a real one. The docstring + below is declared to start at line 10 of `fake.py` (zero-based 9), + so its sixth line, the broken example, is line 15 of that file. The + example documenting a traceback must be tolerated on the way there. + + """ + docstring = ''' + >>> x = 1 + >>> raise ValueError('documented') + Traceback (most recent call last): + ValueError: documented + >>> undefined_name + x + ''' + test = doctest.DocTestParser().get_doctest( + docstring, globs={}, name='synthetic', filename='fake.py', lineno=9) + with pytest.raises(AssertionError) as info: + _run_examples(test) + message = str(info.value) + assert message.startswith( + 'fake.py:15: example 3 of 3 in synthetic raised NameError') + assert '>>> undefined_name + x' in message + assert isinstance(info.value.__cause__, NameError)