From 4dc12344d5013b791636caf86aa4f051d6cbaf72 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:51:34 +0100 Subject: [PATCH 1/2] Hygiene: ship py.typed, fix testpaths, de-duplicate caching.py header Three independent defects on the package that sits at the bottom of the dependency tree, plus a redundant metadata file. 1. PEP 561 marker was missing Without `dol/py.typed`, every downstream `from dol import ...` is type-checked as `Any`. Added the marker and declared the wheel/sdist contents explicitly so it is guaranteed to ship. Verified end-to-end: built the wheel and confirmed `dol/py.typed` is inside it, then type-checked `from dol import Files; reveal_type(Files)` against both the currently published wheel (mypy: "missing library stubs or py.typed marker", revealed type `Any`) and the new one (revealed type `dol.filesys.Files`, no errors). 2. Local pytest and CI pytest ran different suites `testpaths` named a `tests` directory that does not exist, so pytest silently fell back to recursive discovery with doctests OFF: a bare `pytest` collected 189 tests while CI collected ~500. The two runs also used different doctest flags, so local-green did not imply CI-green on the package everything depends on. - `testpaths = ["dol"]` -- the package dir is the test root (unit tests in dol/tests, doctests in the modules). - `addopts = "--doctest-modules --ignore=dol/scrap"` -- one source of truth for what gets collected, honoured by both local and CI runs. `[tool.wads.ci.testing].exclude_paths` is now `[]` accordingly; its old `["examples", "scrap"]` resolved against the repo root and matched nothing, which is why scrap was being collected in CI. - `doctest_optionflags = ["ELLIPSIS", "IGNORE_EXCEPTION_DETAIL"]` -- the exact flags the CI action passes via `-o` (it does NOT pass NORMALIZE_WHITESPACE, so configuring it here would let whitespace- fragile doctests pass locally and fail in CI). No doctests needed fixing: all of them already pass without NORMALIZE_WHITESPACE. Bare `pytest` now reports the same numbers as the CI invocation. 3. dol/caching.py defined its header twice A second import/alias block redefined Instance, PropertyFunc, MethodName and Cache and, more importantly, defined a second `identity` that shadowed the documented one -- so its three doctest examples were never executed despite being collected in CI's mind. Merged the two blocks into a single header (keeping `partial`, `wraps`, `Protocol`, the TypeVars and `_NOT_FOUND`, which the second block uniquely provided) and dropped the duplicates. `dol.caching.identity` is now collected as a live doctest. 4. Deleted setup.cfg, a second metadata source of truth duplicating name/version/description. Both the version-bump and packaging tooling already prefer pyproject.toml and treat setup.cfg as optional legacy. Tests: added dol/tests/test_packaging_hygiene.py (py.typed marker present; testpaths all exist; addopts collects doctests; doctest flags match CI) and header-integrity + live-doctest tests in dol/tests/test_caching.py. All ten fail on the parent commit and pass here. Claude-Session: https://claude.ai/code/session_01Kug7UUbVeCQgruvNXUq63c --- dol/caching.py | 45 ++++--------- dol/py.typed | 0 dol/tests/test_caching.py | 32 +++++++++ dol/tests/test_packaging_hygiene.py | 101 ++++++++++++++++++++++++++++ pyproject.toml | 30 ++++++++- setup.cfg | 27 -------- 6 files changed, 173 insertions(+), 62 deletions(-) create mode 100644 dol/py.typed create mode 100644 dol/tests/test_packaging_hygiene.py delete mode 100644 setup.cfg diff --git a/dol/caching.py b/dol/caching.py index 92952aac..b1847216 100644 --- a/dol/caching.py +++ b/dol/caching.py @@ -66,17 +66,24 @@ import os import types -from typing import Optional, KT, VT, Any, Union, T -from collections.abc import Callable -from collections.abc import Mapping +from functools import RLock, cached_property, partial, wraps +from types import GenericAlias +from typing import Any, Optional, Protocol, TypeVar, Union +from collections.abc import Callable, Mapping, MutableMapping from dol.base import Store from dol.trans import store_decorator -from functools import RLock, cached_property -from types import GenericAlias -from collections.abc import MutableMapping +# Type variables +KT = TypeVar("KT") # Key type +VT = TypeVar("VT") # Value type +T = TypeVar("T") # Generic type + +#: Sentinel marking "no value cached yet". Distinct from every user value, so a +#: legitimately cached ``None`` is not mistaken for a cache miss. +_NOT_FOUND = object() +# Type definitions Instance = Any PropertyFunc = Callable[[Instance], VT] MethodName = str @@ -97,32 +104,6 @@ def identity(x: T) -> T: return x -from functools import RLock, partial, wraps -from types import GenericAlias -from collections.abc import MutableMapping -from typing import Optional, TypeVar, Union, Any, Protocol -from collections.abc import Callable - -# Type variables -KT = TypeVar("KT") # Key type -VT = TypeVar("VT") # Value type -T = TypeVar("T") # Generic type - -# Constants -_NOT_FOUND = object() - -# Type definitions -Instance = Any -PropertyFunc = Callable[[Instance], VT] -MethodName = str -Cache = Union[MethodName, MutableMapping[KT, VT]] - - -def identity(x: T) -> T: - """Identity function that returns its input unchanged.""" - return x - - class KeyStrategy(Protocol): """Protocol defining how a key strategy should behave.""" diff --git a/dol/py.typed b/dol/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/dol/tests/test_caching.py b/dol/tests/test_caching.py index 2a18d818..498cb48e 100644 --- a/dol/tests/test_caching.py +++ b/dol/tests/test_caching.py @@ -3,8 +3,11 @@ import pytest from functools import partial, cached_property from collections import UserDict +from pathlib import Path from typing import Dict, Any +from dol import caching as caching_module + # Import the refactored implementations - adjust the import path as needed from dol.caching import ( cache_this, @@ -1117,5 +1120,34 @@ def g(self): assert cache["g"] == 8, "Value from disk should be in cache" +#: Definitions that must appear exactly once in ``dol/caching.py``. A second, +#: shadowing copy of the module header silently kills the doctests of whatever +#: it shadows (pytest collects the *live* object, not the dead source). +SINGLY_DEFINED_IN_CACHING = ( + "def identity(", + "Instance = Any", + "PropertyFunc = Callable[[Instance], VT]", + "MethodName = str", + "Cache = Union[MethodName, MutableMapping[KT, VT]]", +) + + +@pytest.mark.parametrize("definition", SINGLY_DEFINED_IN_CACHING) +def test_caching_module_header_is_not_duplicated(definition): + """``dol.caching`` must not define its header twice.""" + source = Path(caching_module.__file__).read_text(encoding="utf-8") + count = source.count(definition) + assert count == 1, f"{definition!r} appears {count} times in dol/caching.py" + + +def test_identity_keeps_its_doctests(): + """The live ``dol.caching.identity`` must be the documented one. + + A second, undocumented ``identity`` further down the module used to shadow + the documented one, so its three doctests were never actually run. + """ + assert ">>> identity(42)" in caching_module.identity.__doc__ + + if __name__ == "__main__": pytest.main(["-xvs", __file__]) diff --git a/dol/tests/test_packaging_hygiene.py b/dol/tests/test_packaging_hygiene.py new file mode 100644 index 00000000..9633de65 --- /dev/null +++ b/dol/tests/test_packaging_hygiene.py @@ -0,0 +1,101 @@ +"""Packaging and test-configuration hygiene checks for the ``dol`` distribution. + +``dol`` sits at the bottom of a large dependency tree, so a few "boring" +packaging properties are worth asserting explicitly instead of trusting by +inspection: + +- The :pep:`561` ``py.typed`` marker must be present *and shipped*. Without it, + every ``from dol import ...`` in a downstream package is typed as ``Any``. +- The pytest configuration must point at paths that actually exist and enable + the same doctest collection CI uses. Otherwise a green local ``pytest`` says + nothing about a green CI run, on the very package everything depends on. +""" + +from pathlib import Path + +import pytest + +import dol + +#: Name of the PEP 561 marker file that tells type checkers the package is typed. +PY_TYPED_MARKER_NAME = "py.typed" + +#: Doctest option flags the wads ``run-tests-uv`` CI action passes via ``-o``. +#: Local ``doctest_optionflags`` must agree with these, or local-green does not +#: imply CI-green (notably, CI does *not* pass ``NORMALIZE_WHITESPACE``). +CI_DOCTEST_OPTIONFLAGS = frozenset({"ELLIPSIS", "IGNORE_EXCEPTION_DETAIL"}) + +#: pytest ``addopts`` entries required for a bare local ``pytest`` to collect +#: what CI collects. +REQUIRED_ADDOPTS = ("--doctest-modules",) + +#: Package root directory (works both for editable and installed distributions). +PKG_DIR = Path(dol.__file__).parent + + +def _load_toml(path: Path) -> dict: + """Parse a TOML file, skipping the test if no TOML parser is available.""" + try: + import tomllib # Python >= 3.11 + except ModuleNotFoundError: # pragma: no cover - Python 3.10 without tomli + try: + import tomli as tomllib + except ModuleNotFoundError: + pytest.skip("No TOML parser available (need Python >= 3.11 or tomli)") + return tomllib.loads(path.read_text(encoding="utf-8")) + + +def _pytest_ini_options() -> dict: + """Return ``[tool.pytest.ini_options]``, skipping if there's no source tree. + + Installed (non-editable) distributions have no ``pyproject.toml`` next to + the package, so the configuration tests are source-tree-only. + """ + pyproject = PKG_DIR.parent / "pyproject.toml" + if not pyproject.is_file(): + pytest.skip("Not running from a source tree (no pyproject.toml)") + return _load_toml(pyproject).get("tool", {}).get("pytest", {}).get("ini_options", {}) + + +def test_py_typed_marker_is_present(): + """The PEP 561 marker must exist inside the package directory.""" + marker = PKG_DIR / PY_TYPED_MARKER_NAME + assert marker.is_file(), ( + f"Missing {PY_TYPED_MARKER_NAME} in {PKG_DIR.name}/: without it every " + "downstream `from dol import ...` is type-checked as Any." + ) + + +def test_testpaths_all_exist(): + """Every configured ``testpaths`` entry must resolve to a real path. + + A ``testpaths`` pointing at a nonexistent directory makes pytest silently + fall back to recursive discovery from the cwd, so what a bare ``pytest`` + runs bears no relation to what is configured. + """ + ini_options = _pytest_ini_options() + project_root = PKG_DIR.parent + testpaths = ini_options.get("testpaths", []) + missing = [p for p in testpaths if not (project_root / p).exists()] + assert not missing, f"testpaths entries do not exist: {missing}" + + +def test_addopts_collect_doctests_like_ci(): + """A bare ``pytest`` must collect doctests, as CI does.""" + addopts = _pytest_ini_options().get("addopts", "") + missing = [opt for opt in REQUIRED_ADDOPTS if opt not in addopts] + assert not missing, f"pytest addopts is missing {missing} (got {addopts!r})" + + +def test_doctest_optionflags_match_ci(): + """Local doctest flags must match the flags the CI action passes. + + The wads ``run-tests-uv`` action passes + ``-o doctest_optionflags='ELLIPSIS IGNORE_EXCEPTION_DETAIL'``, which + *overrides* the ini value. Configuring anything else locally (e.g. adding + ``NORMALIZE_WHITESPACE``) means locally-passing doctests can fail in CI. + """ + flags = frozenset(_pytest_ini_options().get("doctest_optionflags", [])) + assert flags == CI_DOCTEST_OPTIONFLAGS, ( + f"doctest_optionflags {sorted(flags)} != CI's {sorted(CI_DOCTEST_OPTIONFLAGS)}" + ) diff --git a/pyproject.toml b/pyproject.toml index 2fe0b8ef..8b10e64c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,17 @@ requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.build.targets.wheel] +# Declared explicitly (rather than inferred) so the distributed contents are a +# stated fact. `artifacts` guarantees the PEP 561 marker ships even if a future +# ignore rule would otherwise exclude it -- without `dol/py.typed` in the wheel, +# every downstream `from dol import ...` is type-checked as `Any`. +packages = ["dol"] +artifacts = ["dol/py.typed"] + +[tool.hatch.build.targets.sdist] +artifacts = ["dol/py.typed"] + [project] name = "dol" version = "0.3.57" @@ -49,8 +60,18 @@ convention = "google" [tool.pytest.ini_options] minversion = "6.0" -testpaths = ["tests"] -doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"] +# The package dir IS the test root: unit tests live in dol/tests and doctests +# live in the modules themselves. (There is no top-level `tests` dir; naming one +# made pytest silently fall back to recursive discovery and skip all doctests.) +testpaths = ["dol"] +# addopts is the single source of truth for what gets collected, so a bare local +# `pytest` collects exactly what CI collects. Hence `exclude_paths = []` below. +addopts = "--doctest-modules --ignore=dol/scrap" +# Must match the flags the wads `run-tests-uv` action passes via `-o` (which +# override this ini value). Notably CI does NOT pass NORMALIZE_WHITESPACE, so +# configuring it here would let whitespace-fragile doctests pass locally and +# fail in CI. +doctest_optionflags = ["ELLIPSIS", "IGNORE_EXCEPTION_DETAIL"] [tool.wads.ci] project_name = "" @@ -82,7 +103,10 @@ pytest_args = ["-v", "--tb=short"] coverage_enabled = true coverage_threshold = 0 coverage_report_format = ["term", "xml"] -exclude_paths = ["examples", "scrap"] +# Empty on purpose: exclusions are declared once, in pytest's `addopts` above, +# so local and CI runs collect the same set. (The old ["examples", "scrap"] was +# resolved against the repo root and therefore matched nothing.) +exclude_paths = [] test_on_windows = true [tool.wads.ci.build] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 78240dc5..00000000 --- a/setup.cfg +++ /dev/null @@ -1,27 +0,0 @@ -[metadata] -name = dol -version = 0.3.57 -url = https://github.com/i2mint/dol -platforms = any -description_file = README.md -description = Base builtin tools make and transform data object layers (dols). -root_url = https://github.com/i2mint -license = mit -long_description = file:README.md -long_description_content_type = text/markdown -keywords = - storage - interface -display_name = dol - -[options] -packages = find: -include_package_data = True -zip_safe = False -tests_require = - numpy -install_requires = - -[options.extras_require] -testing = - From a5c16b88f77394cd557c11ecbeaf7dccabebabda Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:59:16 +0100 Subject: [PATCH 2/2] docs: note that a bare pytest now equals the CI run Claude-Session: https://claude.ai/code/session_01Kug7UUbVeCQgruvNXUq63c --- CLAUDE.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index b4fcb83b..7c11a0ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -193,7 +193,12 @@ from dol import Files s = wrap_kvs(Files("/data"), obj_of_data=json.loads, data_of_obj=json.dumps) ``` -Run tests: `pytest dol/tests/` +Run tests: a bare `pytest` (from the repo root) runs exactly what CI runs — the +`dol/tests/` unit tests **and** every module doctest, with CI's doctest flags. +Narrow it with `pytest dol/tests/` (unit tests only) or `pytest dol/caching.py` +(one module's doctests). Do not add `NORMALIZE_WHITESPACE` to +`doctest_optionflags`: CI does not pass it, so doctests relying on it would pass +locally and fail in CI. ---