From 99709b3a497d2821067c2aee0a85a4701a980fb8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 00:39:08 +0000 Subject: [PATCH 01/14] Refuse stale build/lib leftovers and manifest-added package data at build time setuptools reuses build/lib between builds and removes nothing from it, so a module left there by an install from another commit was installed by the next install even though src/ does not contain it, and tests/test_install_contract.py still passed. A stale src/universal_law_query.egg-info/SOURCES.txt could also add an untracked file as package data. Both reproduced in isolated copies with pip 26.2.1, setuptools 77.0.3 and wheel 0.45.1 (pip install --target, pip wheel and an isolated PEP 517 build). - setup.py: a build_py subclass compares the whole build_lib tree with its own outputs after building and refuses (REFUSED: stale build output ...) when anything else is present, naming the files and the documented git clean remedy. Nothing is deleted. Editable installs run from src/ and are unaffected. - pyproject.toml: include-package-data = false; the package ships no package data, so a stale manifest cannot add any. - tests/test_install_provenance.py: temporary-copy controls for clean and deliberately contaminated installs (pip install --target, pip wheel, opt-in isolated build), installed-file inventory against the tracked package files, stale egg-info data, and a repeated clean install. - README.md: the leftover paragraph now describes the refusal. Scientific effect: NONE. No runtime dependency; lookup, verification and wrapper behaviour unchanged; the custom source archive contract is unaffected (setup.py is not an archive member). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 25 ++++- pyproject.toml | 4 + setup.py | 48 ++++++++ tests/test_install_provenance.py | 184 +++++++++++++++++++++++++++++++ 4 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 setup.py create mode 100644 tests/test_install_provenance.py diff --git a/README.md b/README.md index cde3652..e7840a5 100644 --- a/README.md +++ b/README.md @@ -46,12 +46,25 @@ Installing from the checkout, and the installation control in `tests/test_install_contract.py`, leave setuptools' `build/` and `src/universal_law_query.egg-info/` in the checkout. Both are listed in `.gitignore`, so `git status` stays clean, and the source dry-run builder (which -selects tracked files only) is unaffected. They are not harmless: a file left -in `build/lib/` (for example by an install from another commit) is installed by -the next install even though `src/` does not contain it, and the installation -control still passes. Remove them before installing: -`git clean -fdX -- build src/universal_law_query.egg-info`. A fresh checkout, -as in CI, has neither directory. +selects tracked files only) is unaffected. They are not harmless: setuptools +reuses `build/lib/` without removing anything, so a file left there (for +example by an install from another commit) would otherwise be installed by the +next install even though `src/` does not contain it, and a stale `SOURCES.txt` +in the egg-info directory would otherwise add package data; the installation +control above cannot see either. The checkout's `setup.py` therefore refuses to +build when `build/lib/` holds anything this source tree did not produce +(`REFUSED: stale build output …`, naming the files), and `pyproject.toml` +declares that the package ships no package data. Nothing is deleted for you; +remove the leftovers before installing: +`git clean -fdX -- build src/universal_law_query.egg-info`. +`tests/test_install_provenance.py` installs temporary copies of the checkout, +clean and deliberately contaminated, and checks that the installed files are +exactly the tracked package files. A fresh checkout, as in CI, has neither +directory. An untracked file placed directly in `src/universal_law_query/` is +source, not a leftover, and is installed like any other module. Editable +installs (`pip install -e`) run from `src/` and are unaffected. The custom +source archive below does not carry `setup.py`; it has no leftovers of its own, +and installation from it is not one of its offline checks. The canonical implementation lives in `src/universal_law_query/`. Historical root commands — `research_query.py`, `catalog_entry_helper.py`, and `verify_portable_stubs.py` — remain thin compatibility wrappers so existing `python -B -S` workflows continue to work. diff --git a/pyproject.toml b/pyproject.toml index efb7e23..373ba38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,5 +22,9 @@ Issues = "https://github.com/d6g8k5htny-coder/query-/issues" [project.scripts] universal-law-query = "universal_law_query.cli:main" +[tool.setuptools] +# The package ships no package data; a stale egg-info manifest must not add any. +include-package-data = false + [tool.setuptools.packages.find] where = ["src"] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..0920327 --- /dev/null +++ b/setup.py @@ -0,0 +1,48 @@ +"""Build-time leftover guard for installs from this checkout. + +Scientific effect: NONE. This file adds no runtime dependency and changes no +lookup, verification or refusal behaviour of the package. + +setuptools reuses ``build/lib`` between builds and removes nothing from it. A +module left there by an install from another commit is copied into the next +wheel although ``src/`` does not contain it, and an import check of the +installed package still passes. This ``build_py`` compares the whole +``build_lib`` tree with the files it has just produced and refuses to continue +when anything else is present. Nothing is deleted; the refusal names the +leftovers and the documented remedy. +""" +from __future__ import annotations + +import os + +from setuptools import setup +from setuptools.command.build_py import build_py as _build_py +from setuptools.errors import SetupError + +REMEDY = 'git clean -fdX -- build src/universal_law_query.egg-info' + + +def _canonical(path: str) -> str: + return os.path.normcase(os.path.normpath(os.path.abspath(path))) + + +class build_py(_build_py): + """Refuse build output that this source tree did not produce.""" + + def run(self) -> None: + super().run() + produced = {_canonical(path) for path in self.get_outputs(include_bytecode=False)} + stale = [] + for dirpath, _dirnames, filenames in os.walk(self.build_lib): + for name in filenames: + path = os.path.join(dirpath, name) + if _canonical(path) not in produced: + stale.append(os.path.relpath(path).replace(os.sep, '/')) + if stale: + raise SetupError( + 'REFUSED: stale build output not produced from this source tree: ' + + ', '.join(sorted(stale)) + + '. Remove build leftovers before installing: ' + REMEDY) + + +setup(cmdclass={'build_py': build_py}) diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py new file mode 100644 index 0000000..498c699 --- /dev/null +++ b/tests/test_install_provenance.py @@ -0,0 +1,184 @@ +"""Installed-module provenance controls for installs from this checkout. + +Scientific effect: NONE. These controls exercise packaging only. + +Every case installs from a temporary copy of the checkout's tracked and +untracked-but-not-ignored files, so the real checkout's ``build/`` and +``egg-info`` directories are never read or written. The contaminated cases +plant files under the copy's ``build/lib/`` the way an install from another +commit leaves them, then require the build to refuse rather than install them. +""" +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE = 'universal_law_query' +SOURCE_DIR = Path('src') / PACKAGE +STALE_FILES = ('stale_leftover.py', 'stale_pkg/__init__.py') +PIP_TIMEOUT = 180 + + +def tracked_source_files() -> list[str]: + raw = subprocess.check_output( + ['git', '-C', str(ROOT), 'ls-files', '-z', '--', SOURCE_DIR.as_posix()], timeout=10) + return sorted(Path(name).relative_to(SOURCE_DIR).as_posix() + for name in raw.decode().split('\0') if name) + + +def copy_checkout(dest: Path) -> None: + # Tracked plus untracked-but-not-ignored files: an uncommitted repair is + # exercised, while the real checkout's ignored build/ and egg-info are not. + raw = subprocess.check_output( + ['git', '-C', str(ROOT), 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], + timeout=10) + for name in raw.decode().split('\0'): + if not name: + continue + parts = Path(name).parts + if '__pycache__' in parts or name.endswith(('.pyc', '.pyo')): + continue + source = ROOT / name + if not source.is_file() or source.is_symlink(): + continue + target = dest / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def plant_leftovers(checkout: Path) -> list[Path]: + planted = [] + for name in STALE_FILES: + path = checkout / 'build' / 'lib' / PACKAGE / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('STALE = True # left by an install from another commit\n', encoding='utf-8') + planted.append(path) + return planted + + +def pip(*args: str) -> subprocess.CompletedProcess: + env = dict(os.environ) + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' + env['PYTHONDONTWRITEBYTECODE'] = '1' + return subprocess.run([sys.executable, '-m', 'pip', *args], env=env, + capture_output=True, text=True, timeout=PIP_TIMEOUT) + + +def installed_files(target: Path) -> list[str]: + package = target / PACKAGE + if not package.is_dir(): + return [] + return sorted(path.relative_to(package).as_posix() for path in package.rglob('*') + if path.is_file() and '__pycache__' not in path.parts) + + +def wheel_package_files(wheel: Path) -> list[str]: + with zipfile.ZipFile(wheel) as archive: + return sorted(name[len(PACKAGE) + 1:] for name in archive.namelist() + if name.startswith(PACKAGE + '/')) + + +class InstallProvenance(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.base = Path(self.tmp.name) + self.checkout = self.base / 'checkout' + self.checkout.mkdir() + copy_checkout(self.checkout) + self.target = self.base / 'site' + self.target.mkdir() + self.wheels = self.base / 'wheels' + self.wheels.mkdir() + self.assertFalse((self.checkout / 'build').exists(), 'copy must start without build/') + self.assertTrue(tracked_source_files(), 'tracked package inventory must be non-empty') + + def install(self): + return pip('install', '--quiet', '--no-deps', '--no-build-isolation', + '--target', str(self.target), str(self.checkout)) + + def wheel(self): + return pip('wheel', '--quiet', '--no-deps', '--no-build-isolation', + '--wheel-dir', str(self.wheels), str(self.checkout)) + + def test_clean_copy_installs_exactly_the_tracked_package_files(self): + result = self.install() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(installed_files(self.target), tracked_source_files()) + probe = subprocess.run( + [sys.executable, '-c', 'import universal_law_query, sys; print(universal_law_query.__file__)'], + env={'PYTHONPATH': str(self.target)}, capture_output=True, text=True, timeout=10) + self.assertEqual(probe.returncode, 0, probe.stderr) + self.assertTrue(Path(probe.stdout.strip()).resolve().is_relative_to(self.target.resolve())) + + def test_clean_copy_wheel_holds_exactly_the_tracked_package_files(self): + result = self.wheel() + self.assertEqual(result.returncode, 0, result.stderr) + wheels = sorted(self.wheels.glob('*.whl')) + self.assertEqual(len(wheels), 1, wheels) + self.assertEqual(wheel_package_files(wheels[0]), tracked_source_files()) + + def test_stale_build_lib_leftovers_are_refused_and_not_installed(self): + plant_leftovers(self.checkout) + result = self.install() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'install must fail closed on build/lib leftovers') + self.assertIn('REFUSED: stale build output', output) + for name in STALE_FILES: + self.assertIn('build/lib/' + PACKAGE + '/' + name, output) + self.assertIn('git clean -fdX -- build src/universal_law_query.egg-info', output) + self.assertEqual(installed_files(self.target), [], 'nothing may be installed after a refusal') + # The leftovers are reported, not deleted. + for name in STALE_FILES: + self.assertTrue((self.checkout / 'build' / 'lib' / PACKAGE / name).is_file()) + + def test_stale_build_lib_leftovers_block_the_wheel(self): + plant_leftovers(self.checkout) + result = self.wheel() + self.assertNotEqual(result.returncode, 0, 'wheel build must fail closed on build/lib leftovers') + self.assertIn('REFUSED: stale build output', result.stdout + result.stderr) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + + def test_stale_egg_info_manifest_cannot_add_package_data(self): + # A SOURCES.txt left by an earlier build is read back when no revision + # control plugin is present. The package ships no package data, so no + # manifest entry may add a file to the installed package. + egg_info = self.checkout / 'src' / 'universal_law_query.egg-info' + egg_info.mkdir() + (self.checkout / SOURCE_DIR / 'stale_data.txt').write_text('stale data\n', encoding='utf-8') + (egg_info / 'SOURCES.txt').write_text( + 'README.md\npyproject.toml\nsrc/universal_law_query/__init__.py\n' + 'src/universal_law_query/stale_data.txt\n', encoding='utf-8') + result = self.install() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(installed_files(self.target), tracked_source_files()) + + def test_second_clean_install_after_a_first_is_not_refused(self): + # The guard must not mistake this build's own earlier output for leftovers. + first = self.install() + self.assertEqual(first.returncode, 0, first.stderr) + shutil.rmtree(self.target) + self.target.mkdir() + second = self.install() + self.assertEqual(second.returncode, 0, second.stderr) + self.assertEqual(installed_files(self.target), tracked_source_files()) + + @unittest.skipUnless(os.environ.get('QUERY_INSTALL_ISOLATED') == '1', + 'isolated build downloads setuptools; set QUERY_INSTALL_ISOLATED=1 to run') + def test_isolated_build_also_refuses_stale_build_lib_leftovers(self): + plant_leftovers(self.checkout) + result = pip('install', '--quiet', '--no-deps', '--target', str(self.target), str(self.checkout)) + self.assertNotEqual(result.returncode, 0) + self.assertIn('REFUSED: stale build output', result.stdout + result.stderr) + self.assertEqual(installed_files(self.target), []) + + +if __name__ == '__main__': + unittest.main() From 18d6128a7729037752a1953736cc5301954ed277 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 01:59:00 +0000 Subject: [PATCH 02/14] Count the build's own bytecode as produced; narrow the README untracked-file sentence Author amendment for the two non-blocking limits in the nonauthor review of query-#46 at 99709b3a (xAI Grok 4.7, Cursor agent bc-1ceb4665). - setup.py: compare build/lib against get_outputs(include_bytecode=True), so the direct `python setup.py build_py --compile` (or -O) path no longer refuses the __pycache__ files the same run has just written. Stale bytecode for a module this source tree does not produce, or for another interpreter tag, is still refused. The pip path writes no bytecode and is unchanged. - tests/test_install_provenance.py: new control for the direct path: a first and a repeated `build_py --compile` succeed and leave bytecode under build/lib; planted leftovers are still refused there, and the refusal names no .pyc. The case failed against the previous guard (it named five .pyc files) and passes now. - README.md: the untracked-source sentence is limited to .py modules; other untracked files are not installed because the package declares no package data; one sentence records the direct-path bytecode behaviour. Scientific effect NONE. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 7 +++++-- setup.py | 5 +++-- tests/test_install_provenance.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e7840a5..22a7376 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,11 @@ remove the leftovers before installing: `tests/test_install_provenance.py` installs temporary copies of the checkout, clean and deliberately contaminated, and checks that the installed files are exactly the tracked package files. A fresh checkout, as in CI, has neither -directory. An untracked file placed directly in `src/universal_law_query/` is -source, not a leftover, and is installed like any other module. Editable +directory. An untracked `.py` module placed directly in `src/universal_law_query/` +is source, not a leftover, and is installed like any other module; other +untracked files there are not installed, because the package declares no +package data. The direct `python setup.py build_py --compile` path counts the +bytecode it writes in the same run as its own output. Editable installs (`pip install -e`) run from `src/` and are unaffected. The custom source archive below does not carry `setup.py`; it has no leftovers of its own, and installation from it is not one of its offline checks. diff --git a/setup.py b/setup.py index 0920327..5e0604c 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,8 @@ wheel although ``src/`` does not contain it, and an import check of the installed package still passes. This ``build_py`` compares the whole ``build_lib`` tree with the files it has just produced and refuses to continue -when anything else is present. Nothing is deleted; the refusal names the +when anything else is present. Bytecode written by the same run (``--compile`` +or ``-O``) counts as produced. Nothing is deleted; the refusal names the leftovers and the documented remedy. """ from __future__ import annotations @@ -31,7 +32,7 @@ class build_py(_build_py): def run(self) -> None: super().run() - produced = {_canonical(path) for path in self.get_outputs(include_bytecode=False)} + produced = {_canonical(path) for path in self.get_outputs(include_bytecode=True)} stale = [] for dirpath, _dirnames, filenames in os.walk(self.build_lib): for name in filenames: diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index 498c699..7eb8d63 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -71,6 +71,14 @@ def pip(*args: str) -> subprocess.CompletedProcess: capture_output=True, text=True, timeout=PIP_TIMEOUT) +def direct_build_py(checkout: Path, *args: str) -> subprocess.CompletedProcess: + # The legacy direct command path. Bytecode writing stays enabled so that the + # build's own ``__pycache__`` output is part of what the guard inspects. + env = {key: value for key, value in os.environ.items() if key != 'PYTHONDONTWRITEBYTECODE'} + return subprocess.run([sys.executable, 'setup.py', 'build_py', *args], cwd=checkout, env=env, + capture_output=True, text=True, timeout=PIP_TIMEOUT) + + def installed_files(target: Path) -> list[str]: package = target / PACKAGE if not package.is_dir(): @@ -170,6 +178,26 @@ def test_second_clean_install_after_a_first_is_not_refused(self): self.assertEqual(second.returncode, 0, second.stderr) self.assertEqual(installed_files(self.target), tracked_source_files()) + def test_direct_build_py_compile_accepts_its_own_bytecode(self): + # ``python setup.py build_py --compile`` writes ``__pycache__`` under + # build/lib in the same run. That output is this build's, not a leftover, + # so neither the first run nor a repeat may be refused; planted leftovers + # are still refused on this path. + first = direct_build_py(self.checkout, '--compile') + self.assertEqual(first.returncode, 0, first.stdout + first.stderr) + pycs = sorted((self.checkout / 'build' / 'lib' / PACKAGE / '__pycache__').glob('*.pyc')) + self.assertTrue(pycs, 'the compile step must have written bytecode under build/lib') + second = direct_build_py(self.checkout, '--compile') + self.assertEqual(second.returncode, 0, second.stdout + second.stderr) + plant_leftovers(self.checkout) + third = direct_build_py(self.checkout, '--compile') + self.assertNotEqual(third.returncode, 0, 'leftovers must still be refused on the direct path') + output = third.stdout + third.stderr + self.assertIn('REFUSED: stale build output', output) + for name in STALE_FILES: + self.assertIn('build/lib/' + PACKAGE + '/' + name, output) + self.assertNotIn('.pyc', output, 'this build\'s own bytecode must not be named as a leftover') + @unittest.skipUnless(os.environ.get('QUERY_INSTALL_ISOLATED') == '1', 'isolated build downloads setuptools; set QUERY_INSTALL_ISOLATED=1 to run') def test_isolated_build_also_refuses_stale_build_lib_leftovers(self): From 9a7d71456fa47e39e6c5834ce2c7b20ae23d10b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 02:13:48 +0000 Subject: [PATCH 03/14] Record the implicit typing-file exception and the direct-path cache admission Documentation-only author amendment for the two non-blocking limits in the re-read of query-#46 at 18d6128a (xAI Grok 4.7, Cursor agent bc-1ceb4665). - README.md: the untracked-file sentence now names the typing files setuptools always includes (py.typed, *.pyi) as the exception, and states that the direct `build_py --compile` path admits its cache paths by name, so a file already at one of them is not refused there while a following pip install still refuses it. - setup.py: the module docstring records the same cache-path admission. No code or test change. Scientific effect NONE. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 7 +++++-- setup.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 22a7376..e424979 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,11 @@ exactly the tracked package files. A fresh checkout, as in CI, has neither directory. An untracked `.py` module placed directly in `src/universal_law_query/` is source, not a leftover, and is installed like any other module; other untracked files there are not installed, because the package declares no -package data. The direct `python setup.py build_py --compile` path counts the -bytecode it writes in the same run as its own output. Editable +package data, except the typing files setuptools always includes (`py.typed`, +`*.pyi`). The direct `python setup.py build_py --compile` path counts the +bytecode it writes in the same run as its own output; it admits those cache +paths by name, so a file already sitting at one of them is not refused there +(a following `pip install` still refuses it). Editable installs (`pip install -e`) run from `src/` and are unaffected. The custom source archive below does not carry `setup.py`; it has no leftovers of its own, and installation from it is not one of its offline checks. diff --git a/setup.py b/setup.py index 5e0604c..df8e02a 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,11 @@ installed package still passes. This ``build_py`` compares the whole ``build_lib`` tree with the files it has just produced and refuses to continue when anything else is present. Bytecode written by the same run (``--compile`` -or ``-O``) counts as produced. Nothing is deleted; the refusal names the -leftovers and the documented remedy. +or ``-O``) counts as produced; that admission is by cache path, so a file +already at a path this run would write is accepted even when the byte-compile +step leaves its bytes in place (the pip path writes no bytecode and refuses +such a file). Nothing is deleted; the refusal names the leftovers and the +documented remedy. """ from __future__ import annotations From d51f8740a5cb628e44cd33019e5ca80e8923cd94 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 10:10:27 +0000 Subject: [PATCH 04/14] Guard the wheel staging tree, --skip-build, symlinks, newer stale bytes and the egg-info manifest Author amendment for the six findings of the automatic reviews of query-#46 at 9a7d7145 (Codex reviews 5581785458 and 5581856876), each reproduced first on temporary copies with setuptools 77.0.3: - a leftover build/bdist.*/wheel staging tree was reused and archived; - `setup.py bdist_wheel --skip-build` never ran build_py, so the guard was bypassed; - a symlinked directory under build/lib passed the walk and was dereferenced at staging (and a symlinked file at a produced path survived the copy); - a stale SOURCES.txt still fed `setup.py sdist`; - a leftover at a produced path newer than its source was kept as "up to date" and shipped; - the remedy named the checkout's build/ even for a custom --build-lib. setup.py: one shared walk refuses every file not produced and every symlink; build_py checks before and after its run, forces every copy and compares each produced module's bytes with its source; a bdist_wheel subclass refuses any file under its staging tree and repeats the build/lib check (comparing bytes under --skip-build); an egg_info subclass regenerates SOURCES.txt without reading a leftover manifest back; the remedy names a custom build directory. Nothing is deleted. tests/test_install_provenance.py: eight new controls, red against the previous setup.py and green now (16 tests, one opt-in skip); discovery 102 OK, legacy 38 OK, -O OK, isolated opt-in OK. README paragraph updated. Scientific effect NONE. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 25 ++-- setup.py | 160 ++++++++++++++++++++----- tests/test_install_provenance.py | 195 +++++++++++++++++++++++++++++-- 3 files changed, 335 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index e424979..6c735f8 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,16 @@ next install even though `src/` does not contain it, and a stale `SOURCES.txt` in the egg-info directory would otherwise add package data; the installation control above cannot see either. The checkout's `setup.py` therefore refuses to build when `build/lib/` holds anything this source tree did not produce -(`REFUSED: stale build output …`, naming the files), and `pyproject.toml` -declares that the package ships no package data. Nothing is deleted for you; -remove the leftovers before installing: -`git clean -fdX -- build src/universal_law_query.egg-info`. +(`REFUSED: stale build output …`, naming the files), before copying anything +and again afterwards; every symlink under `build/lib/` is refused wherever it +points, because the wheel staging step would follow it. The wheel command +refuses a leftover staging tree under `build/bdist.*/wheel/` (an interrupted +build leaves one) and repeats the `build/lib/` check, so `--skip-build` cannot +bypass it. The egg-info manifest is regenerated from the tree and a stale +`SOURCES.txt` is never read back, so it can add a file neither to a wheel nor +to a `setup.py sdist` archive; `pyproject.toml` also declares that the package +ships no package data. Nothing is deleted for you; remove the leftovers before +installing: `git clean -fdX -- build src/universal_law_query.egg-info`. `tests/test_install_provenance.py` installs temporary copies of the checkout, clean and deliberately contaminated, and checks that the installed files are exactly the tracked package files. A fresh checkout, as in CI, has neither @@ -64,10 +70,13 @@ directory. An untracked `.py` module placed directly in `src/universal_law_query is source, not a leftover, and is installed like any other module; other untracked files there are not installed, because the package declares no package data, except the typing files setuptools always includes (`py.typed`, -`*.pyi`). The direct `python setup.py build_py --compile` path counts the -bytecode it writes in the same run as its own output; it admits those cache -paths by name, so a file already sitting at one of them is not refused there -(a following `pip install` still refuses it). Editable +`*.pyi`). The build rewrites every file it produces and then compares each +module's bytes with its source, so a file already sitting at a path this build +writes (a module from another commit, or bytecode under `--compile`) is +replaced, never shipped; `setup.py bdist_wheel --skip-build` compares instead +of rebuilding and refuses differing bytes. A refusal in a custom `--build-lib` +directory names that directory, since the `git clean` remedy reaches only the +checkout's `build/`. Editable installs (`pip install -e`) run from `src/` and are unaffected. The custom source archive below does not carry `setup.py`; it has no leftovers of its own, and installation from it is not one of its offline checks. diff --git a/setup.py b/setup.py index df8e02a..dd2ee39 100644 --- a/setup.py +++ b/setup.py @@ -3,24 +3,43 @@ Scientific effect: NONE. This file adds no runtime dependency and changes no lookup, verification or refusal behaviour of the package. -setuptools reuses ``build/lib`` between builds and removes nothing from it. A -module left there by an install from another commit is copied into the next -wheel although ``src/`` does not contain it, and an import check of the -installed package still passes. This ``build_py`` compares the whole -``build_lib`` tree with the files it has just produced and refuses to continue -when anything else is present. Bytecode written by the same run (``--compile`` -or ``-O``) counts as produced; that admission is by cache path, so a file -already at a path this run would write is accepted even when the byte-compile -step leaves its bytes in place (the pip path writes no bytecode and refuses -such a file). Nothing is deleted; the refusal names the leftovers and the -documented remedy. +setuptools reuses ``build/`` between builds and removes nothing from it. A +module left under ``build/lib`` by an install from another commit is copied +into the next wheel although ``src/`` does not contain it, and an import check +of the installed package still passes. Three commands are guarded: + +- ``build_py`` compares the whole ``build/lib`` tree with the files it + produces, before copying anything and again afterwards, and refuses when + anything else is present. Every symlink under the tree is refused wherever it + points, because the walk does not follow symlinks but the wheel staging step + dereferences them. The run rewrites every file it produces (setuptools would + otherwise skip a destination that is newer than its source), and afterwards + each produced module's bytes are compared with its source, so bytes left at a + path this run writes are replaced, never shipped. Bytecode written by the same + run (``--compile`` or ``-O``) counts as produced and is recompiled. +- ``bdist_wheel`` refuses a leftover staging tree under ``build/bdist.*/wheel`` + (an interrupted build leaves one, and the next build archives whatever it + holds) and repeats the ``build/lib`` comparison, so ``--skip-build`` cannot + bypass it; with ``--skip-build`` nothing is rebuilt, so the byte comparison + runs there instead. +- ``egg_info`` regenerates ``SOURCES.txt`` from the tree and never reads a + leftover manifest back in, so a stale entry can add a file neither to a wheel + nor to a source distribution. + +Nothing is deleted; each refusal names the leftovers and the remedy, which is +``git clean`` for the checkout's own ``build/`` and otherwise names the +configured directory. """ from __future__ import annotations +import filecmp import os from setuptools import setup +from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools.command.build_py import build_py as _build_py +from setuptools.command.egg_info import egg_info as _egg_info +from setuptools.command.egg_info import manifest_maker as _manifest_maker from setuptools.errors import SetupError REMEDY = 'git clean -fdX -- build src/universal_law_query.egg-info' @@ -30,23 +49,112 @@ def _canonical(path: str) -> str: return os.path.normcase(os.path.normpath(os.path.abspath(path))) +def _relative(path: str) -> str: + return os.path.relpath(path).replace(os.sep, '/') + + +def _unexpected(tree: str, produced: set[str]) -> list[str]: + """Files under ``tree`` that are not in ``produced``, and every symlink.""" + found = [] + if os.path.islink(tree): + return [_relative(tree) + ' (symlink)'] + if not os.path.isdir(tree): + return found + for dirpath, dirnames, filenames in os.walk(tree): + for name in dirnames: + if os.path.islink(os.path.join(dirpath, name)): + found.append(_relative(os.path.join(dirpath, name)) + ' (symlink)') + for name in filenames: + path = os.path.join(dirpath, name) + if os.path.islink(path): + found.append(_relative(path) + ' (symlink)') + elif _canonical(path) not in produced: + found.append(_relative(path)) + return sorted(found) + + +def _remedy(tree: str) -> str: + """The checkout's own build/ is cleaned by git; another directory is named.""" + try: + relative = os.path.relpath(tree) + except ValueError: # another drive on Windows + relative = None + if relative is not None and (relative == 'build' or relative.startswith('build' + os.sep)): + return REMEDY + return ('remove the named files under ' + _relative(tree) + + ' yourself; it is outside the checkout\'s build/, so ' + REMEDY + ' does not reach it') + + +def _refuse(unexpected: list[str], tree: str) -> None: + if unexpected: + raise SetupError( + 'REFUSED: stale build output not produced from this source tree: ' + + ', '.join(unexpected) + + '. Remove build leftovers before installing: ' + _remedy(tree)) + + +def _differing_outputs(build_py_command) -> list[str]: + """Produced module and data paths whose bytes are not this source tree's.""" + pairs = [] + for package, module, module_file in build_py_command.find_all_modules(): + output = build_py_command.get_module_outfile(build_py_command.build_lib, package.split('.'), module) + pairs.append((module_file, output)) + for _package, src_dir, build_dir, filenames in build_py_command.data_files: + for filename in filenames: + pairs.append((os.path.join(src_dir, filename), os.path.join(build_dir, filename))) + found = [] + for source, output in pairs: + if os.path.isfile(output) and not filecmp.cmp(source, output, shallow=False): + found.append(_relative(output) + ' (bytes differ from ' + _relative(source) + ')') + return found + + +def _guard_build_lib(build_py_command, compare_bytes: bool = False) -> None: + produced = {_canonical(path) for path in build_py_command.get_outputs(include_bytecode=True)} + unexpected = _unexpected(build_py_command.build_lib, produced) + if compare_bytes: + unexpected = sorted(unexpected + _differing_outputs(build_py_command)) + _refuse(unexpected, build_py_command.build_lib) + + class build_py(_build_py): """Refuse build output that this source tree did not produce.""" def run(self) -> None: + _guard_build_lib(self) + # Rewrite every output: a leftover at a produced path that is newer than + # its source would otherwise be kept as "up to date" and shipped. + self.force = True super().run() - produced = {_canonical(path) for path in self.get_outputs(include_bytecode=True)} - stale = [] - for dirpath, _dirnames, filenames in os.walk(self.build_lib): - for name in filenames: - path = os.path.join(dirpath, name) - if _canonical(path) not in produced: - stale.append(os.path.relpath(path).replace(os.sep, '/')) - if stale: - raise SetupError( - 'REFUSED: stale build output not produced from this source tree: ' - + ', '.join(sorted(stale)) - + '. Remove build leftovers before installing: ' + REMEDY) - - -setup(cmdclass={'build_py': build_py}) + _guard_build_lib(self, compare_bytes=True) + + +class bdist_wheel(_bdist_wheel): + """Refuse a leftover staging tree, and re-check build/lib even with --skip-build.""" + + def run(self) -> None: + _refuse(_unexpected(self.bdist_dir, set()), self.bdist_dir) + # Without --skip-build the build rewrites build/lib next; with it, the + # bytes already there are what gets shipped, so compare them now. + _guard_build_lib(self.get_finalized_command('build_py'), compare_bytes=bool(self.skip_build)) + super().run() + + +class _fresh_manifest_maker(_manifest_maker): + """Build SOURCES.txt from the tree; a leftover manifest is never read back.""" + + def read_manifest(self) -> None: + return None + + +class egg_info(_egg_info): + def find_sources(self) -> None: + manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') + mm = _fresh_manifest_maker(self.distribution) + mm.ignore_egg_info_dir = getattr(self, 'ignore_egg_info_in_manifest', False) + mm.manifest = manifest_filename + mm.run() + self.filelist = mm.filelist + + +setup(cmdclass={'build_py': build_py, 'bdist_wheel': bdist_wheel, 'egg_info': egg_info}) diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index 7eb8d63..b6e8540 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -5,8 +5,10 @@ Every case installs from a temporary copy of the checkout's tracked and untracked-but-not-ignored files, so the real checkout's ``build/`` and ``egg-info`` directories are never read or written. The contaminated cases -plant files under the copy's ``build/lib/`` the way an install from another -commit leaves them, then require the build to refuse rather than install them. +plant leftovers the way an earlier build leaves them (files and symlinks under +the copy's ``build/lib/``, a wheel staging tree under ``build/bdist.*/wheel/``, +a stale egg-info manifest), then require the build to refuse rather than +install them, or the manifest to be regenerated rather than read back. """ from __future__ import annotations @@ -14,6 +16,7 @@ import shutil import subprocess import sys +import tarfile import tempfile import unittest import zipfile @@ -79,6 +82,46 @@ def direct_build_py(checkout: Path, *args: str) -> subprocess.CompletedProcess: capture_output=True, text=True, timeout=PIP_TIMEOUT) +def direct_setup(checkout: Path, *args: str) -> subprocess.CompletedProcess: + # Any other legacy direct command (``bdist_wheel``, ``sdist``), without bytecode. + env = dict(os.environ) + env['PYTHONDONTWRITEBYTECODE'] = '1' + return subprocess.run([sys.executable, 'setup.py', *args], cwd=checkout, env=env, + capture_output=True, text=True, timeout=PIP_TIMEOUT) + + +def wheel_staging_dir(checkout: Path) -> Path: + # The directory bdist_wheel stages the archive in (``build/bdist./wheel``), + # asked of setuptools itself so the platform tag matches the build's. + code = ('from setuptools.dist import Distribution\n' + 'from setuptools.command.bdist_wheel import bdist_wheel\n' + 'command = bdist_wheel(Distribution())\n' + 'command.finalize_options()\n' + 'print(command.bdist_dir)\n') + raw = subprocess.check_output([sys.executable, '-c', code], cwd=checkout, text=True, timeout=60) + return checkout / raw.strip() + + +def plant_stale_manifest(checkout: Path) -> Path: + # A SOURCES.txt left by an earlier build, naming an untracked file that exists. + egg_info = checkout / 'src' / 'universal_law_query.egg-info' + egg_info.mkdir() + stale = checkout / SOURCE_DIR / 'stale_data.txt' + stale.write_text('stale data\n', encoding='utf-8') + (egg_info / 'SOURCES.txt').write_text( + 'README.md\npyproject.toml\nsrc/universal_law_query/__init__.py\n' + 'src/universal_law_query/stale_data.txt\n', encoding='utf-8') + return egg_info / 'SOURCES.txt' + + +def sdist_members(dist_dir: Path) -> list[str]: + tarballs = sorted(dist_dir.glob('*.tar.gz')) + if len(tarballs) != 1: + raise AssertionError(f'expected one sdist, found {tarballs}') + with tarfile.open(tarballs[0]) as archive: + return sorted(name.split('/', 1)[1] for name in archive.getnames() if '/' in name) + + def installed_files(target: Path) -> list[str]: package = target / PACKAGE if not package.is_dir(): @@ -155,19 +198,149 @@ def test_stale_build_lib_leftovers_block_the_wheel(self): self.assertEqual(sorted(self.wheels.glob('*.whl')), []) def test_stale_egg_info_manifest_cannot_add_package_data(self): - # A SOURCES.txt left by an earlier build is read back when no revision - # control plugin is present. The package ships no package data, so no - # manifest entry may add a file to the installed package. - egg_info = self.checkout / 'src' / 'universal_law_query.egg-info' - egg_info.mkdir() - (self.checkout / SOURCE_DIR / 'stale_data.txt').write_text('stale data\n', encoding='utf-8') - (egg_info / 'SOURCES.txt').write_text( - 'README.md\npyproject.toml\nsrc/universal_law_query/__init__.py\n' - 'src/universal_law_query/stale_data.txt\n', encoding='utf-8') + # A SOURCES.txt left by an earlier build would be read back when no + # revision control plugin is present. The package ships no package data, + # so no manifest entry may add a file to the installed package. + plant_stale_manifest(self.checkout) result = self.install() self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(installed_files(self.target), tracked_source_files()) + def test_sdist_never_reads_a_stale_manifest_back(self): + # ``include-package-data = false`` keeps manifest entries out of a wheel + # but not out of a source distribution, so the manifest itself must be + # regenerated from the tree instead of accumulating leftover entries. + clean = self.base / 'clean' + clean.mkdir() + copy_checkout(clean) + reference = direct_setup(clean, 'sdist', '--dist-dir', str(self.base / 'clean_dist')) + self.assertEqual(reference.returncode, 0, reference.stdout + reference.stderr) + manifest = plant_stale_manifest(self.checkout) + result = direct_setup(self.checkout, 'sdist', '--dist-dir', str(self.base / 'dist')) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + members = sdist_members(self.base / 'dist') + self.assertEqual(members, sdist_members(self.base / 'clean_dist')) + self.assertNotIn('src/universal_law_query/stale_data.txt', members) + self.assertNotIn('stale_data.txt', manifest.read_text(encoding='utf-8'), + 'the regenerated manifest must not carry the stale entry') + + def test_stale_wheel_staging_tree_is_refused(self): + # An interrupted wheel build leaves build/bdist./wheel behind; + # the next build would archive whatever that tree holds. + staged = wheel_staging_dir(self.checkout) / PACKAGE / 'stage_stale.py' + staged.parent.mkdir(parents=True) + staged.write_text('STAGE_STALE = True\n', encoding='utf-8') + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'wheel build must fail closed on a staging leftover') + self.assertIn('REFUSED: stale build output', output) + self.assertIn(staged.relative_to(self.checkout).as_posix(), output) + self.assertIn('git clean -fdX -- build src/universal_law_query.egg-info', output) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + self.assertTrue(staged.is_file(), 'the leftover is reported, not deleted') + + def test_direct_bdist_wheel_skip_build_still_refuses_stale_build_lib(self): + # ``--skip-build`` never runs build_py, so the wheel command must repeat + # the inventory check itself. + stale = self.checkout / 'build' / 'lib' / PACKAGE / 'stale.py' + stale.parent.mkdir(parents=True) + stale.write_text('STALE = True\n', encoding='utf-8') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_wheel', '--skip-build', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a skipped build must not bypass the guard') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('build/lib/' + PACKAGE + '/stale.py', output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + + def test_direct_bdist_wheel_skip_build_after_a_clean_build_is_accepted(self): + built = direct_setup(self.checkout, 'build_py') + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_wheel', '--skip-build', '--dist-dir', str(dist)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + wheels = sorted(dist.glob('*.whl')) + self.assertEqual(len(wheels), 1, wheels) + self.assertEqual(wheel_package_files(wheels[0]), tracked_source_files()) + + def test_newer_stale_bytes_at_a_produced_path_are_rewritten_not_shipped(self): + # setuptools skips a destination that is newer than its source, so a + # leftover module at a path this build writes would be kept as is. The + # build rewrites every output and then compares bytes with the source. + planted = self.checkout / 'build' / 'lib' / PACKAGE / 'catalog.py' + planted.parent.mkdir(parents=True) + planted.write_text('PLANTED_MARKER = True\n', encoding='utf-8') + future = os.stat(self.checkout / SOURCE_DIR / 'catalog.py').st_mtime + 86400 + os.utime(planted, (future, future)) + result = self.wheel() + self.assertEqual(result.returncode, 0, result.stderr) + wheels = sorted(self.wheels.glob('*.whl')) + self.assertEqual(len(wheels), 1, wheels) + with zipfile.ZipFile(wheels[0]) as archive: + shipped = archive.read(PACKAGE + '/catalog.py') + self.assertEqual(shipped, (self.checkout / SOURCE_DIR / 'catalog.py').read_bytes()) + self.assertNotIn(b'PLANTED_MARKER', shipped) + + def test_direct_bdist_wheel_skip_build_refuses_differing_bytes(self): + built = direct_setup(self.checkout, 'build_py') + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + stale = self.checkout / 'build' / 'lib' / PACKAGE / 'catalog.py' + stale.write_text('PLANTED_MARKER = True\n', encoding='utf-8') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_wheel', '--skip-build', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a skipped build must not ship bytes the source tree did not produce') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('build/lib/' + PACKAGE + '/catalog.py (bytes differ from src/' + PACKAGE + '/catalog.py)', output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + + def test_refusal_in_a_custom_build_lib_names_that_directory(self): + # ``git clean -fdX -- build …`` only reaches the checkout's build/; a + # refusal elsewhere must point at the directory it inspected. + outside = self.base / 'elsewhere' + (outside / PACKAGE).mkdir(parents=True) + (outside / PACKAGE / 'stale.py').write_text('STALE = True\n', encoding='utf-8') + result = direct_setup(self.checkout, 'build_py', '--build-lib', str(outside)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0) + self.assertIn('REFUSED: stale build output', output) + relative = os.path.relpath(outside, self.checkout).replace(os.sep, '/') + self.assertIn(relative + '/' + PACKAGE + '/stale.py', output) + self.assertIn('outside the checkout', output) + self.assertIn('under ' + relative, output) + + def test_symlinks_under_build_lib_are_refused(self): + # os.walk does not follow symlinks, but the wheel staging step + # dereferences them, so a symlinked directory or file is a leftover + # wherever it points, even at a path this build would write. + outside = self.base / 'outside' / 'stale_pkg' + outside.mkdir(parents=True) + (outside / '__init__.py').write_text('OUTSIDE = True\n', encoding='utf-8') + build_lib = self.checkout / 'build' / 'lib' / PACKAGE + build_lib.mkdir(parents=True) + try: + os.symlink(outside, build_lib / 'stale_pkg', target_is_directory=True) + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked directory must be refused') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('build/lib/' + PACKAGE + '/stale_pkg (symlink)', output) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + self.assertTrue((build_lib / 'stale_pkg').is_symlink(), 'the symlink is reported, not removed') + # A symlinked file at a path this build writes is refused before the copy. + (build_lib / 'stale_pkg').unlink() + victim = self.base / 'victim.py' + victim.write_text('ORIGINAL = True\n', encoding='utf-8') + os.symlink(victim, build_lib / 'cli.py') + result = direct_setup(self.checkout, 'build_py') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked file must be refused') + self.assertIn('build/lib/' + PACKAGE + '/cli.py (symlink)', output) + self.assertEqual(victim.read_text(encoding='utf-8'), 'ORIGINAL = True\n', + 'nothing may be written through the symlink') + def test_second_clean_install_after_a_first_is_not_refused(self): # The guard must not mistake this build's own earlier output for leftovers. first = self.install() From 0289fbbf5d4083aab11db2da4b17719cef3f8aaa Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 11:24:14 +0000 Subject: [PATCH 05/14] Refuse unrecompiled bytecode, missing outputs, egg-info leftovers, a reused sdist tree and symlinked sources Second round of the build-leftover guard in setup.py, answering the review findings on d51f8740: - Bytecode under build/lib is admitted only when this run recompiles it: not under --skip-build (bdist_wheel, and now install_lib for the legacy install, install_lib and bdist_egg paths), and not when setuptools skips the compile step under python -B or PYTHONDONTWRITEBYTECODE. - Under --skip-build every expected output must be present as a regular file; a missing or irregular output is refused instead of shipping an incomplete wheel. - egg_info refuses any file beside the ones its writers and SOURCES.txt produce, since sdist grafts the whole directory and a direct bdist_wheel copies it into the dist-info. - sdist refuses a leftover release tree (universal_law_query-0.1.0/) before filling it. - A symlink among the package sources in src/, or a symlinked directory above one, is refused before anything is copied; the copy and the byte comparison both follow the link. - The wheel staging check also reports empty leftover directories, which otherwise fail later with a bare FileExistsError. - _relative() tolerates a path on another Windows drive. Controls in tests/test_install_provenance.py: seven new or extended cases, six of which fail at d51f8740 and pass here; the compile-then-skip-build case documents behaviour that already held. The temporary base directory is resolved so expectations match setup.py's cwd-relative paths behind a symlinked TMPDIR, and the vacuous write-through assertion is replaced by an is_symlink check. README paragraph updated. Scientific effect: NONE. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 27 ++-- setup.py | 177 +++++++++++++++++++++------ tests/test_install_provenance.py | 203 +++++++++++++++++++++++++++++-- 3 files changed, 356 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 6c735f8..b4a83f2 100644 --- a/README.md +++ b/README.md @@ -55,14 +55,25 @@ control above cannot see either. The checkout's `setup.py` therefore refuses to build when `build/lib/` holds anything this source tree did not produce (`REFUSED: stale build output …`, naming the files), before copying anything and again afterwards; every symlink under `build/lib/` is refused wherever it -points, because the wheel staging step would follow it. The wheel command -refuses a leftover staging tree under `build/bdist.*/wheel/` (an interrupted -build leaves one) and repeats the `build/lib/` check, so `--skip-build` cannot -bypass it. The egg-info manifest is regenerated from the tree and a stale -`SOURCES.txt` is never read back, so it can add a file neither to a wheel nor -to a `setup.py sdist` archive; `pyproject.toml` also declares that the package -ships no package data. Nothing is deleted for you; remove the leftovers before -installing: `git clean -fdX -- build src/universal_law_query.egg-info`. +points, because the wheel staging step would follow it, and so is a symlink +among the package sources in `src/`, because the copy would ship its target. +Bytecode under `build/lib/` is accepted only from the run that compiles it +(`build_py --compile` with bytecode writing enabled); a cache file found by a +run that does not recompile it, such as `--skip-build` or a build under +`python -B`, is refused. The wheel command refuses a leftover staging tree +under `build/bdist.*/wheel/` (an interrupted build leaves one) and repeats the +`build/lib/` check, so `--skip-build` cannot bypass it; with `--skip-build` +every expected output must be present with the bytes of its source. The +legacy `install`, `install_lib` and `bdist_egg` commands with `--skip-build` +run the same check. The egg-info manifest is regenerated from the tree and a +stale `SOURCES.txt` is never read back, so it can add a file neither to a wheel +nor to a `setup.py sdist` archive; any other file left in the egg-info +directory is refused, because an sdist and a direct `bdist_wheel` would copy +it into the archive, and so is a leftover sdist release tree +(`universal_law_query-0.1.0/`), which an sdist would archive whole. +`pyproject.toml` also declares that the package ships no package data. Nothing +is deleted for you; remove the leftovers before installing: +`git clean -fdX -- build src/universal_law_query.egg-info`. `tests/test_install_provenance.py` installs temporary copies of the checkout, clean and deliberately contaminated, and checks that the installed files are exactly the tracked package files. A fresh checkout, as in CI, has neither diff --git a/setup.py b/setup.py index dd2ee39..fcdd314 100644 --- a/setup.py +++ b/setup.py @@ -6,43 +6,62 @@ setuptools reuses ``build/`` between builds and removes nothing from it. A module left under ``build/lib`` by an install from another commit is copied into the next wheel although ``src/`` does not contain it, and an import check -of the installed package still passes. Three commands are guarded: - -- ``build_py`` compares the whole ``build/lib`` tree with the files it - produces, before copying anything and again afterwards, and refuses when - anything else is present. Every symlink under the tree is refused wherever it - points, because the walk does not follow symlinks but the wheel staging step +of the installed package still passes. Five commands are guarded: + +- ``build_py`` refuses a symlink among the package sources it would copy, or a + symlinked directory above one, because the copy and the byte comparison + below both follow the link and would ship its target as a file of this tree. + It then compares the whole ``build/lib`` tree with the files it produces, + before copying anything and again afterwards, and refuses when anything else + is present. Every symlink under the tree is refused wherever it points, + because the walk does not follow symlinks but the wheel staging step dereferences them. The run rewrites every file it produces (setuptools would otherwise skip a destination that is newer than its source), and afterwards each produced module's bytes are compared with its source, so bytes left at a - path this run writes are replaced, never shipped. Bytecode written by the same - run (``--compile`` or ``-O``) counts as produced and is recompiled. + path this run writes are replaced, never shipped. Bytecode counts as produced + only when this run recompiles it (``--compile`` or ``-O`` with bytecode + writing enabled); under ``python -B`` or ``PYTHONDONTWRITEBYTECODE`` setuptools + skips the compile step, so a cache file found under ``build/lib`` is refused. - ``bdist_wheel`` refuses a leftover staging tree under ``build/bdist.*/wheel`` (an interrupted build leaves one, and the next build archives whatever it holds) and repeats the ``build/lib`` comparison, so ``--skip-build`` cannot - bypass it; with ``--skip-build`` nothing is rebuilt, so the byte comparison - runs there instead. + bypass it. With ``--skip-build`` nothing is rebuilt, so the byte comparison + runs there instead, every expected output must be present as a regular file, + and bytecode, which nothing recompiles, is refused. +- ``install_lib`` applies the same skipped-build comparison, which covers the + legacy ``install``, ``install_lib`` and ``bdist_egg`` commands with + ``--skip-build``. - ``egg_info`` regenerates ``SOURCES.txt`` from the tree and never reads a leftover manifest back in, so a stale entry can add a file neither to a wheel - nor to a source distribution. + nor to a source distribution; any other file left beside the ones this run + writes is refused, because ``sdist`` and a direct ``bdist_wheel`` copy the + whole directory into the archive. +- ``sdist`` refuses a leftover release tree (``universal_law_query-0.1.0/``, + kept by ``--keep-temp`` or an interrupted run), which setuptools would + otherwise fill, archive whole and only then remove. Nothing is deleted; each refusal names the leftovers and the remedy, which is -``git clean`` for the checkout's own ``build/`` and otherwise names the -configured directory. +``git clean`` for the checkout's own ``build/`` and egg-info directory and +otherwise names the directory. """ from __future__ import annotations import filecmp import os +import sys +from importlib import metadata as _metadata from setuptools import setup from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools.command.build_py import build_py as _build_py from setuptools.command.egg_info import egg_info as _egg_info from setuptools.command.egg_info import manifest_maker as _manifest_maker +from setuptools.command.install_lib import install_lib as _install_lib +from setuptools.command.sdist import sdist as _sdist from setuptools.errors import SetupError -REMEDY = 'git clean -fdX -- build src/universal_law_query.egg-info' +EGG_INFO = 'src/universal_law_query.egg-info' +REMEDY = 'git clean -fdX -- build ' + EGG_INFO def _canonical(path: str) -> str: @@ -50,17 +69,24 @@ def _canonical(path: str) -> str: def _relative(path: str) -> str: - return os.path.relpath(path).replace(os.sep, '/') + try: + relative = os.path.relpath(path) + except ValueError: # another drive on Windows: no relative form exists + relative = os.path.abspath(path) + return relative.replace(os.sep, '/') -def _unexpected(tree: str, produced: set[str]) -> list[str]: - """Files under ``tree`` that are not in ``produced``, and every symlink.""" +def _unexpected(tree: str, produced: set[str], empty_dirs: bool = False) -> list[str]: + """Files under ``tree`` that are not in ``produced``, every symlink, and + with ``empty_dirs`` every empty directory below the tree itself.""" found = [] if os.path.islink(tree): return [_relative(tree) + ' (symlink)'] if not os.path.isdir(tree): return found for dirpath, dirnames, filenames in os.walk(tree): + if empty_dirs and dirpath != tree and not dirnames and not filenames: + found.append(_relative(dirpath) + ' (empty directory)') for name in dirnames: if os.path.islink(os.path.join(dirpath, name)): found.append(_relative(os.path.join(dirpath, name)) + ' (symlink)') @@ -74,14 +100,12 @@ def _unexpected(tree: str, produced: set[str]) -> list[str]: def _remedy(tree: str) -> str: - """The checkout's own build/ is cleaned by git; another directory is named.""" - try: - relative = os.path.relpath(tree) - except ValueError: # another drive on Windows - relative = None - if relative is not None and (relative == 'build' or relative.startswith('build' + os.sep)): - return REMEDY - return ('remove the named files under ' + _relative(tree) + """The checkout's own build/ and egg-info are cleaned by git; another directory is named.""" + relative = _relative(tree) + for cleaned in ('build', EGG_INFO): + if relative == cleaned or relative.startswith(cleaned + '/'): + return REMEDY + return ('remove the named files under ' + relative + ' yourself; it is outside the checkout\'s build/, so ' + REMEDY + ' does not reach it') @@ -93,8 +117,8 @@ def _refuse(unexpected: list[str], tree: str) -> None: + '. Remove build leftovers before installing: ' + _remedy(tree)) -def _differing_outputs(build_py_command) -> list[str]: - """Produced module and data paths whose bytes are not this source tree's.""" +def _source_pairs(build_py_command) -> list[tuple[str, str]]: + """(source, output) for every module and data file the build copies.""" pairs = [] for package, module, module_file in build_py_command.find_all_modules(): output = build_py_command.get_module_outfile(build_py_command.build_lib, package.split('.'), module) @@ -102,21 +126,69 @@ def _differing_outputs(build_py_command) -> list[str]: for _package, src_dir, build_dir, filenames in build_py_command.data_files: for filename in filenames: pairs.append((os.path.join(src_dir, filename), os.path.join(build_dir, filename))) + return pairs + + +def _symlinked_sources(build_py_command) -> list[str]: + """Package sources, and directories above them, that are symlinks.""" + found = set() + for source, _output in _source_pairs(build_py_command): + path = os.path.normpath(source) + while path and path != os.path.dirname(path): + if os.path.islink(path): + found.add(_relative(path) + ' (symlink)') + path = os.path.dirname(path) + return sorted(found) + + +def _refuse_sources(symlinked: list[str]) -> None: + if symlinked: + raise SetupError( + 'REFUSED: symlinked package source, which would ship the link\'s target rather than a file of ' + 'this source tree: ' + ', '.join(symlinked) + + '. Replace the symlink with a regular file before installing.') + + +def _differing_outputs(build_py_command) -> list[str]: + """Produced module and data paths that are missing, not regular files, or + whose bytes are not this source tree's.""" found = [] - for source, output in pairs: - if os.path.isfile(output) and not filecmp.cmp(source, output, shallow=False): + for source, output in _source_pairs(build_py_command): + if os.path.islink(output): + continue # already reported by the tree walk + if not os.path.exists(output): + found.append(_relative(output) + ' (missing)') + elif not os.path.isfile(output): + found.append(_relative(output) + ' (not a regular file)') + elif not filecmp.cmp(source, output, shallow=False): found.append(_relative(output) + ' (bytes differ from ' + _relative(source) + ')') return found -def _guard_build_lib(build_py_command, compare_bytes: bool = False) -> None: - produced = {_canonical(path) for path in build_py_command.get_outputs(include_bytecode=True)} +def _guard_build_lib(build_py_command, compare_bytes: bool = False, recompiles: bool = True) -> None: + """Refuse a symlinked source, then anything under build/lib that is not + this source tree's output. + + Bytecode is admitted only when this run rewrites it: ``recompiles`` is + false when nothing is rebuilt (``--skip-build``), and setuptools skips the + compile step under ``python -B`` or ``PYTHONDONTWRITEBYTECODE``. With + ``compare_bytes`` each produced module's and data file's bytes are also + checked against its source, and every expected output must be present. + """ + _refuse_sources(_symlinked_sources(build_py_command)) + admit_bytecode = recompiles and not sys.dont_write_bytecode + produced = {_canonical(path) for path in build_py_command.get_outputs(include_bytecode=admit_bytecode)} unexpected = _unexpected(build_py_command.build_lib, produced) if compare_bytes: unexpected = sorted(unexpected + _differing_outputs(build_py_command)) _refuse(unexpected, build_py_command.build_lib) +def _guard_skipped_build(command) -> None: + """A command that ships build/lib without rebuilding it compares the bytes it has.""" + _guard_build_lib(command.get_finalized_command('build_py'), compare_bytes=True, recompiles=False) + + class build_py(_build_py): """Refuse build output that this source tree did not produce.""" @@ -133,10 +205,21 @@ class bdist_wheel(_bdist_wheel): """Refuse a leftover staging tree, and re-check build/lib even with --skip-build.""" def run(self) -> None: - _refuse(_unexpected(self.bdist_dir, set()), self.bdist_dir) - # Without --skip-build the build rewrites build/lib next; with it, the - # bytes already there are what gets shipped, so compare them now. - _guard_build_lib(self.get_finalized_command('build_py'), compare_bytes=bool(self.skip_build)) + _refuse(_unexpected(self.bdist_dir, set(), empty_dirs=True), self.bdist_dir) + if self.skip_build: + _guard_skipped_build(self) + else: + # The build rewrites build/lib next and checks it again itself. + _guard_build_lib(self.get_finalized_command('build_py')) + super().run() + + +class install_lib(_install_lib): + """The legacy install, install_lib and bdist_egg paths with --skip-build copy build/lib as is.""" + + def run(self) -> None: + if self.skip_build: + _guard_skipped_build(self) super().run() @@ -148,6 +231,8 @@ def read_manifest(self) -> None: class egg_info(_egg_info): + """Regenerate the manifest, and refuse any leftover beside the files this run writes.""" + def find_sources(self) -> None: manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') mm = _fresh_manifest_maker(self.distribution) @@ -156,5 +241,25 @@ def find_sources(self) -> None: mm.run() self.filelist = mm.filelist + def run(self) -> None: + super().run() + # sdist grafts the whole egg-info directory and a direct bdist_wheel copies + # it into the dist-info, so a file no writer of this run produced is a + # leftover with a path into the archive. + written = {entry.name for entry in _metadata.entry_points(group='egg_info.writers')} + written.add('SOURCES.txt') + leftovers = [_relative(os.path.join(self.egg_info, name)) + for name in sorted(os.listdir(self.egg_info)) if name not in written] + _refuse(leftovers, self.egg_info) + + +class sdist(_sdist): + """Refuse a leftover release tree; setuptools would fill it, archive it whole and only then remove it.""" + + def make_release_tree(self, base_dir, files) -> None: + _refuse(_unexpected(base_dir, set()), base_dir) + super().make_release_tree(base_dir, files) + -setup(cmdclass={'build_py': build_py, 'bdist_wheel': bdist_wheel, 'egg_info': egg_info}) +setup(cmdclass={'build_py': build_py, 'bdist_wheel': bdist_wheel, 'install_lib': install_lib, + 'egg_info': egg_info, 'sdist': sdist}) diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index b6e8540..cdc8a99 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -5,13 +5,18 @@ Every case installs from a temporary copy of the checkout's tracked and untracked-but-not-ignored files, so the real checkout's ``build/`` and ``egg-info`` directories are never read or written. The contaminated cases -plant leftovers the way an earlier build leaves them (files and symlinks under -the copy's ``build/lib/``, a wheel staging tree under ``build/bdist.*/wheel/``, -a stale egg-info manifest), then require the build to refuse rather than -install them, or the manifest to be regenerated rather than read back. +plant leftovers the way an earlier build leaves them (files, bytecode and +symlinks under the copy's ``build/lib/``, a wheel staging tree under +``build/bdist.*/wheel/``, a stale egg-info manifest or an extra egg-info file, +an sdist release tree, a symlink among the package sources), then require the +build to refuse rather than install them, or the manifest to be regenerated +rather than read back. The post-run byte comparison inside ``build_py`` is +covered only indirectly: ``force`` already rewrites every output, and the +``--skip-build`` cases exercise the comparison itself. """ from __future__ import annotations +import importlib.util import os import shutil import subprocess @@ -114,6 +119,15 @@ def plant_stale_manifest(checkout: Path) -> Path: return egg_info / 'SOURCES.txt' +def release_tree_name(checkout: Path) -> str: + # The directory sdist stages the archive in (``-``), asked of + # setuptools itself. + result = direct_setup(checkout, '--fullname') + if result.returncode != 0 or not result.stdout.strip(): + raise AssertionError(result.stdout + result.stderr) + return result.stdout.strip().splitlines()[-1] + + def sdist_members(dist_dir: Path) -> list[str]: tarballs = sorted(dist_dir.glob('*.tar.gz')) if len(tarballs) != 1: @@ -140,7 +154,10 @@ class InstallProvenance(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - self.base = Path(self.tmp.name) + # Resolved, so that expectations derived from these paths match the + # cwd-relative paths setup.py reports even when the temporary directory + # itself sits behind a symlink. + self.base = Path(self.tmp.name).resolve() self.checkout = self.base / 'checkout' self.checkout.mkdir() copy_checkout(self.checkout) @@ -238,6 +255,18 @@ def test_stale_wheel_staging_tree_is_refused(self): self.assertIn('git clean -fdX -- build src/universal_law_query.egg-info', output) self.assertEqual(sorted(self.wheels.glob('*.whl')), []) self.assertTrue(staged.is_file(), 'the leftover is reported, not deleted') + # An empty leftover directory is refused with the same message rather + # than failing later, when the dist-info copy finds it in the way. + staged.unlink() + empty = staged.parent.parent / (PACKAGE + '-0.1.0.dist-info') + empty.mkdir() + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'an empty staging directory must be refused as a leftover') + self.assertIn('REFUSED: stale build output', output) + self.assertIn(empty.relative_to(self.checkout).as_posix() + ' (empty directory)', output) + self.assertIn(staged.parent.relative_to(self.checkout).as_posix() + ' (empty directory)', output) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) def test_direct_bdist_wheel_skip_build_still_refuses_stale_build_lib(self): # ``--skip-build`` never runs build_py, so the wheel command must repeat @@ -338,8 +367,8 @@ def test_symlinks_under_build_lib_are_refused(self): output = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, 'a symlinked file must be refused') self.assertIn('build/lib/' + PACKAGE + '/cli.py (symlink)', output) - self.assertEqual(victim.read_text(encoding='utf-8'), 'ORIGINAL = True\n', - 'nothing may be written through the symlink') + self.assertTrue((build_lib / 'cli.py').is_symlink(), 'the symlink is reported, not replaced') + self.assertEqual(victim.read_text(encoding='utf-8'), 'ORIGINAL = True\n') def test_second_clean_install_after_a_first_is_not_refused(self): # The guard must not mistake this build's own earlier output for leftovers. @@ -371,6 +400,166 @@ def test_direct_build_py_compile_accepts_its_own_bytecode(self): self.assertIn('build/lib/' + PACKAGE + '/' + name, output) self.assertNotIn('.pyc', output, 'this build\'s own bytecode must not be named as a leftover') + def test_bytecode_is_refused_when_byte_compiling_is_disabled(self): + # Under ``python -B`` or PYTHONDONTWRITEBYTECODE setuptools skips the + # compile step of ``build_py --compile``, so a cache file found at an + # expected path was written by something else and would be shipped as is. + planted = Path(importlib.util.cache_from_source( + str(self.checkout / 'build' / 'lib' / PACKAGE / 'catalog.py'), optimization='')) + planted.parent.mkdir(parents=True) + planted.write_bytes(b'PLANTED BYTECODE') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'build_py', '--compile', 'bdist_wheel', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'bytecode nothing recompiles must not be admitted') + self.assertIn('REFUSED: stale build output', output) + self.assertIn(planted.relative_to(self.checkout).as_posix(), output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + self.assertEqual(planted.read_bytes(), b'PLANTED BYTECODE', 'the leftover is reported, not deleted') + + def test_skip_build_after_a_compile_build_refuses_the_bytecode(self): + # Bytecode counts as produced only within the run that writes it. A + # later --skip-build wheel recompiles nothing and cannot vouch for the + # cache files it finds, so it refuses them; rebuild without --compile. + built = direct_build_py(self.checkout, '--compile') + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + pycs = sorted((self.checkout / 'build' / 'lib' / PACKAGE / '__pycache__').glob('*.pyc')) + self.assertTrue(pycs, 'the compile step must have written bytecode under build/lib') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_wheel', '--skip-build', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a skipped build must not ship bytecode it cannot vouch for') + self.assertIn('REFUSED: stale build output', output) + for pyc in pycs: + self.assertIn(pyc.relative_to(self.checkout).as_posix(), output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + + def test_direct_bdist_wheel_skip_build_refuses_missing_or_irregular_outputs(self): + # With nothing built, --skip-build would archive a wheel without the + # package; an expected output that is absent, or present but not a + # regular file, is refused. + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_wheel', '--skip-build', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a skipped build with no build output must be refused') + self.assertIn('REFUSED: stale build output', output) + for name in tracked_source_files(): + if name.endswith('.py'): + self.assertIn('build/lib/' + PACKAGE + '/' + name + ' (missing)', output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + built = direct_setup(self.checkout, 'build_py') + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + irregular = self.checkout / 'build' / 'lib' / PACKAGE / 'cli.py' + irregular.unlink() + irregular.mkdir() + result = direct_setup(self.checkout, 'bdist_wheel', '--skip-build', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a directory at a module path must be refused') + self.assertIn('build/lib/' + PACKAGE + '/cli.py (not a regular file)', output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + + def test_leftover_egg_info_files_are_refused(self): + # The manifest maker grafts the whole egg-info directory into an sdist, + # and a direct bdist_wheel copies it into the dist-info, so a file that + # no writer of this run produces has a path into the archive. + egg_info = self.checkout / 'src' / 'universal_law_query.egg-info' + egg_info.mkdir() + leftover = egg_info / 'private_notes.txt' + leftover.write_text('private\n', encoding='utf-8') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'sdist', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'an sdist must not carry a leftover egg-info file') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('src/universal_law_query.egg-info/private_notes.txt', output) + self.assertIn('git clean -fdX -- build src/universal_law_query.egg-info', output) + self.assertEqual(sorted(dist.glob('*.tar.gz')) if dist.exists() else [], []) + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a wheel build regenerates the same egg-info and must refuse too') + self.assertIn('src/universal_law_query.egg-info/private_notes.txt', output) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + self.assertTrue(leftover.is_file(), 'the leftover is reported, not deleted') + + def test_reused_sdist_release_tree_is_refused(self): + # setuptools fills the release tree, archives it whole and removes it + # only afterwards, so a tree kept by --keep-temp or an interrupted run + # would publish whatever it still holds. + release_tree = self.checkout / release_tree_name(self.checkout) + release_tree.mkdir() + leftover = release_tree / 'private.txt' + leftover.write_text('private\n', encoding='utf-8') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'sdist', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'an sdist must not archive a leftover release tree') + self.assertIn('REFUSED: stale build output', output) + self.assertIn(release_tree.name + '/private.txt', output) + self.assertIn('under ' + release_tree.name, output) + self.assertEqual(sorted(dist.glob('*.tar.gz')) if dist.exists() else [], []) + self.assertTrue(leftover.is_file(), 'the leftover is reported, not deleted') + # An empty directory holds nothing to publish and is not a leftover. + leftover.unlink() + result = direct_setup(self.checkout, 'sdist', '--dist-dir', str(dist)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + clean = self.base / 'clean' + clean.mkdir() + copy_checkout(clean) + reference = direct_setup(clean, 'sdist', '--dist-dir', str(self.base / 'clean_dist')) + self.assertEqual(reference.returncode, 0, reference.stdout + reference.stderr) + self.assertEqual(sdist_members(dist), sdist_members(self.base / 'clean_dist')) + + def test_symlinked_package_sources_are_refused(self): + # The copy follows a symlink in src/, and the byte comparison follows it + # too, so the link's target would ship as a module of this tree. Every + # symlinked source, and every symlinked directory above one, is refused + # before anything is copied. + private = self.base / 'private.py' + private.write_text('PRIVATE = True\n', encoding='utf-8') + leak = self.checkout / SOURCE_DIR / 'leak.py' + try: + os.symlink(private, leak) + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked source must be refused') + self.assertIn('REFUSED: symlinked package source', output) + self.assertIn('src/' + PACKAGE + '/leak.py (symlink)', output) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + self.assertFalse((self.checkout / 'build' / 'lib' / PACKAGE / 'leak.py').exists(), + 'nothing may be copied before the refusal') + self.assertTrue(leak.is_symlink(), 'the symlink is reported, not replaced') + leak.unlink() + # A skipped build compares bytes through the link and would find them + # equal; the link itself is refused. + built = direct_setup(self.checkout, 'build_py') + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + catalog = self.checkout / SOURCE_DIR / 'catalog.py' + copy = self.base / 'catalog_copy.py' + shutil.copy2(catalog, copy) + catalog.unlink() + os.symlink(copy, catalog) + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_wheel', '--skip-build', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'equal bytes behind a symlink are still not this tree\'s') + self.assertIn('src/' + PACKAGE + '/catalog.py (symlink)', output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + catalog.unlink() + shutil.copy2(copy, catalog) + # A symlinked package directory is refused through the modules under it. + outside = self.base / 'outside_pkg' + outside.mkdir() + (outside / '__init__.py').write_text('OUTSIDE = True\n', encoding='utf-8') + os.symlink(outside, self.checkout / SOURCE_DIR / 'sub', target_is_directory=True) + result = direct_setup(self.checkout, 'build_py') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked package directory must be refused') + self.assertIn('REFUSED: symlinked package source', output) + self.assertIn('src/' + PACKAGE + '/sub (symlink)', output) + self.assertFalse((self.checkout / 'build' / 'lib' / PACKAGE / 'sub').exists()) + @unittest.skipUnless(os.environ.get('QUERY_INSTALL_ISOLATED') == '1', 'isolated build downloads setuptools; set QUERY_INSTALL_ISOLATED=1 to run') def test_isolated_build_also_refuses_stale_build_lib_leftovers(self): From 83019d82efaf88bacf9161fb37dbf54b91b4ac1a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 11:50:02 +0000 Subject: [PATCH 06/14] Refuse symlinks in or above the egg-info directory and a symlinked build/ before anything is written Answers the nonauthor AMEND at 0289fbbf: egg_info's writers open each metadata file by name and follow a symlink, so a link planted under a writer's name (SOURCES.txt, top_level.txt, ...) had its target overwritten before the post-run allow-list ran. The egg-info directory is now checked before super().run(): the directory itself and every entry must be regular files, and only names the writers or SOURCES.txt produce may be present; the check is repeated afterwards. The same write-through rule covers a symlinked directory above a relative build/lib, the wheel staging tree or the egg-info directory (build/, src/): refused before any write, since every write under it would land wherever the link points. Absolute, deliberately configured paths are taken as given. Controls: test_symlinked_egg_info_entries_are_refused_before_anything_is_written (two writer-named links, then the directory itself as a link; the outside bytes and the links survive; pip wheel refuses too) and test_symlinked_build_directory_is_refused (pip wheel and direct build_py; nothing written through the link). Both fail at 0289fbbf and pass here; provenance 24 OK (also under -O), discovery 113 OK, legacy 38 OK. README sentence added. Scientific effect: NONE. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 5 ++- setup.py | 67 ++++++++++++++++++++++++----- tests/test_install_provenance.py | 72 +++++++++++++++++++++++++++++++- 3 files changed, 132 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b4a83f2..eb6afd3 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,10 @@ stale `SOURCES.txt` is never read back, so it can add a file neither to a wheel nor to a `setup.py sdist` archive; any other file left in the egg-info directory is refused, because an sdist and a direct `bdist_wheel` would copy it into the archive, and so is a leftover sdist release tree -(`universal_law_query-0.1.0/`), which an sdist would archive whole. +(`universal_law_query-0.1.0/`), which an sdist would archive whole. A symlink +in or at the egg-info directory, or a symlinked `build/`, is refused before +anything is written, because setuptools' metadata writers and build copies +would otherwise overwrite whatever the link points at. `pyproject.toml` also declares that the package ships no package data. Nothing is deleted for you; remove the leftovers before installing: `git clean -fdX -- build src/universal_law_query.egg-info`. diff --git a/setup.py b/setup.py index fcdd314..40b41d0 100644 --- a/setup.py +++ b/setup.py @@ -35,11 +35,19 @@ leftover manifest back in, so a stale entry can add a file neither to a wheel nor to a source distribution; any other file left beside the ones this run writes is refused, because ``sdist`` and a direct ``bdist_wheel`` copy the - whole directory into the archive. + whole directory into the archive. The directory is checked before the + writers run as well as after: a writer opens each metadata file by name and + would follow a symlink, overwriting whatever it points at, so the directory + itself and every entry in it must be regular. - ``sdist`` refuses a leftover release tree (``universal_law_query-0.1.0/``, kept by ``--keep-temp`` or an interrupted run), which setuptools would otherwise fill, archive whole and only then remove. +A symlinked directory above any of these trees (``build/`` above ``build/lib`` +and the staging tree, ``src/`` above the egg-info directory) is refused before +anything is written, because every write under it would land wherever the link +points. + Nothing is deleted; each refusal names the leftovers and the remedy, which is ``git clean`` for the checkout's own ``build/`` and egg-info directory and otherwise names the directory. @@ -141,6 +149,41 @@ def _symlinked_sources(build_py_command) -> list[str]: return sorted(found) +def _symlinked_parents(path: str) -> list[str]: + """Directories above a relative ``path`` that are symlinks; a write under + ``path`` would land wherever they point. An absolute path, configured + deliberately, is taken as given.""" + found = [] + if os.path.isabs(path): + return found + parent = os.path.dirname(os.path.normpath(path)) + while parent and parent != os.path.dirname(parent): + if os.path.islink(parent): + found.append(_relative(parent) + ' (symlink)') + parent = os.path.dirname(parent) + return found + + +def _egg_info_leftovers(tree: str, written: set[str]) -> list[str]: + """Everything in the egg-info directory that is not a regular file with a + name this run writes: a symlink anywhere (a writer would follow it and + overwrite its target), a directory, or a file no writer produces.""" + found = _symlinked_parents(tree) + if os.path.islink(tree): + return sorted(found + [_relative(tree) + ' (symlink)']) + if not os.path.isdir(tree): + return sorted(found) + for name in sorted(os.listdir(tree)): + path = os.path.join(tree, name) + if os.path.islink(path): + found.append(_relative(path) + ' (symlink)') + elif not os.path.isfile(path): + found.append(_relative(path) + ' (not a regular file)') + elif name not in written: + found.append(_relative(path)) + return sorted(found) + + def _refuse_sources(symlinked: list[str]) -> None: if symlinked: raise SetupError( @@ -178,7 +221,8 @@ def _guard_build_lib(build_py_command, compare_bytes: bool = False, recompiles: _refuse_sources(_symlinked_sources(build_py_command)) admit_bytecode = recompiles and not sys.dont_write_bytecode produced = {_canonical(path) for path in build_py_command.get_outputs(include_bytecode=admit_bytecode)} - unexpected = _unexpected(build_py_command.build_lib, produced) + unexpected = sorted(_symlinked_parents(build_py_command.build_lib) + + _unexpected(build_py_command.build_lib, produced)) if compare_bytes: unexpected = sorted(unexpected + _differing_outputs(build_py_command)) _refuse(unexpected, build_py_command.build_lib) @@ -205,7 +249,8 @@ class bdist_wheel(_bdist_wheel): """Refuse a leftover staging tree, and re-check build/lib even with --skip-build.""" def run(self) -> None: - _refuse(_unexpected(self.bdist_dir, set(), empty_dirs=True), self.bdist_dir) + _refuse(sorted(_symlinked_parents(self.bdist_dir) + _unexpected(self.bdist_dir, set(), empty_dirs=True)), + self.bdist_dir) if self.skip_build: _guard_skipped_build(self) else: @@ -242,15 +287,17 @@ def find_sources(self) -> None: self.filelist = mm.filelist def run(self) -> None: - super().run() - # sdist grafts the whole egg-info directory and a direct bdist_wheel copies - # it into the dist-info, so a file no writer of this run produced is a - # leftover with a path into the archive. written = {entry.name for entry in _metadata.entry_points(group='egg_info.writers')} written.add('SOURCES.txt') - leftovers = [_relative(os.path.join(self.egg_info, name)) - for name in sorted(os.listdir(self.egg_info)) if name not in written] - _refuse(leftovers, self.egg_info) + # Before the writers run: each opens its file by name and would follow a + # symlink, overwriting the target, so nothing may be written until the + # directory is known to hold only regular files this run rewrites. sdist + # grafts the whole directory and a direct bdist_wheel copies it into the + # dist-info, so a file no writer produces is a leftover with a path into + # the archive; the check is repeated afterwards. + _refuse(_egg_info_leftovers(self.egg_info, written), self.egg_info) + super().run() + _refuse(_egg_info_leftovers(self.egg_info, written), self.egg_info) class sdist(_sdist): diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index cdc8a99..6e3e3b1 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -8,7 +8,8 @@ plant leftovers the way an earlier build leaves them (files, bytecode and symlinks under the copy's ``build/lib/``, a wheel staging tree under ``build/bdist.*/wheel/``, a stale egg-info manifest or an extra egg-info file, -an sdist release tree, a symlink among the package sources), then require the +symlinks in or at the egg-info directory, a symlinked ``build/``, an sdist +release tree, a symlink among the package sources), then require the build to refuse rather than install them, or the manifest to be regenerated rather than read back. The post-run byte comparison inside ``build_py`` is covered only indirectly: ``force`` already rewrites every output, and the @@ -560,6 +561,75 @@ def test_symlinked_package_sources_are_refused(self): self.assertIn('src/' + PACKAGE + '/sub (symlink)', output) self.assertFalse((self.checkout / 'build' / 'lib' / PACKAGE / 'sub').exists()) + def test_symlinked_egg_info_entries_are_refused_before_anything_is_written(self): + # egg_info's writers open each metadata file by name and follow a + # symlink, so a link planted under a writer's name would have its + # target overwritten before any later check. The directory is checked + # first, and the directory itself may not be a link either. + outside = self.base / 'outside' + outside.mkdir() + secret = outside / 'secret.txt' + secret.write_text('SECRET\n', encoding='utf-8') + egg_info = self.checkout / 'src' / 'universal_law_query.egg-info' + egg_info.mkdir() + try: + os.symlink(secret, egg_info / 'SOURCES.txt') + os.symlink(secret, egg_info / 'top_level.txt') + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + result = direct_setup(self.checkout, 'egg_info') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked metadata file must be refused') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('src/universal_law_query.egg-info/SOURCES.txt (symlink)', output) + self.assertIn('src/universal_law_query.egg-info/top_level.txt (symlink)', output) + self.assertIn('git clean -fdX -- build src/universal_law_query.egg-info', output) + self.assertEqual(secret.read_text(encoding='utf-8'), 'SECRET\n', 'nothing may be written through the link') + self.assertTrue((egg_info / 'SOURCES.txt').is_symlink(), 'the link is reported, not replaced') + self.assertFalse((egg_info / 'PKG-INFO').exists(), 'no writer may run before the refusal') + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'the wheel build regenerates the same egg-info and must refuse too') + self.assertIn('src/universal_law_query.egg-info/SOURCES.txt (symlink)', output) + self.assertEqual(secret.read_text(encoding='utf-8'), 'SECRET\n') + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + # The egg-info directory itself as a symlink: nothing may be written + # into the link's target. + shutil.rmtree(egg_info) + os.symlink(outside, egg_info, target_is_directory=True) + result = direct_setup(self.checkout, 'egg_info') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked egg-info directory must be refused') + self.assertIn('src/universal_law_query.egg-info (symlink)', output) + self.assertEqual(sorted(path.name for path in outside.iterdir()), ['secret.txt'], + 'nothing may be written into the link target') + self.assertTrue(egg_info.is_symlink(), 'the link is reported, not replaced') + + def test_symlinked_build_directory_is_refused(self): + # Every write under build/ lands wherever a symlinked build/ points, so + # a symlinked directory above build/lib or the staging tree is refused + # before anything is written. + outside = self.base / 'outside_build' + outside.mkdir() + try: + os.symlink(outside, self.checkout / 'build', target_is_directory=True) + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked build directory must be refused') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('build (symlink)', output) + self.assertIn('git clean -fdX -- build src/universal_law_query.egg-info', output) + self.assertEqual(sorted(outside.iterdir()), [], 'nothing may be written through the link') + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + self.assertTrue((self.checkout / 'build').is_symlink(), 'the link is reported, not replaced') + result = direct_setup(self.checkout, 'build_py') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'the direct build must refuse the same link') + self.assertIn('build (symlink)', output) + self.assertEqual(sorted(outside.iterdir()), []) + @unittest.skipUnless(os.environ.get('QUERY_INSTALL_ISOLATED') == '1', 'isolated build downloads setuptools; set QUERY_INSTALL_ISOLATED=1 to run') def test_isolated_build_also_refuses_stale_build_lib_leftovers(self): From 16fb1e64abd6639099cd7e716448dbdbcbc050a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 12:01:08 +0000 Subject: [PATCH 07/14] Refuse hard-linked egg-info entries and symlinked sdist inputs, validate install_lib's build directory, keep dry runs Answers the automatic review at 0289fbbf (five findings; the symlinked build/ ancestor was already met by 83019d82): - An egg-info entry with more than one link is refused before the writers run: a hard link shares its bytes with another path and would be written through like a symlink. - sdist refuses a symlink among the files the manifest names, or a symlinked directory above one, before the release tree is filled; every entry is copied or hard-linked into the tree and archived. - install_lib --skip-build validates the directory it copies from (--build-dir), with the expected outputs relocated under it, instead of the default build/lib. - build_py skips the post-run byte comparison on --dry-run, which writes nothing; the leftover check before the run still applies. Controls: four new tests, all red at 83019d82 and green here; provenance 28 OK (also under -O), discovery 117 OK, legacy 38 OK. README updated. Scientific effect: NONE. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 8 +-- setup.py | 93 +++++++++++++++++++++----------- tests/test_install_provenance.py | 87 +++++++++++++++++++++++++++++- 3 files changed, 153 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index eb6afd3..e2deb07 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,12 @@ stale `SOURCES.txt` is never read back, so it can add a file neither to a wheel nor to a `setup.py sdist` archive; any other file left in the egg-info directory is refused, because an sdist and a direct `bdist_wheel` would copy it into the archive, and so is a leftover sdist release tree -(`universal_law_query-0.1.0/`), which an sdist would archive whole. A symlink -in or at the egg-info directory, or a symlinked `build/`, is refused before +(`universal_law_query-0.1.0/`), which an sdist would archive whole, and a +symlink among the files an sdist's manifest names. A symlink or hard link in +or at the egg-info directory, or a symlinked `build/`, is refused before anything is written, because setuptools' metadata writers and build copies -would otherwise overwrite whatever the link points at. +would otherwise overwrite whatever the link points at. A `--dry-run` build +writes nothing and is not refused for that; leftovers are still refused. `pyproject.toml` also declares that the package ships no package data. Nothing is deleted for you; remove the leftovers before installing: `git clean -fdX -- build src/universal_law_query.egg-info`. diff --git a/setup.py b/setup.py index 40b41d0..ffadfef 100644 --- a/setup.py +++ b/setup.py @@ -22,14 +22,17 @@ only when this run recompiles it (``--compile`` or ``-O`` with bytecode writing enabled); under ``python -B`` or ``PYTHONDONTWRITEBYTECODE`` setuptools skips the compile step, so a cache file found under ``build/lib`` is refused. + A dry run writes nothing, so the comparison after the run is skipped there; + the leftover check before it still applies. - ``bdist_wheel`` refuses a leftover staging tree under ``build/bdist.*/wheel`` (an interrupted build leaves one, and the next build archives whatever it holds) and repeats the ``build/lib`` comparison, so ``--skip-build`` cannot bypass it. With ``--skip-build`` nothing is rebuilt, so the byte comparison runs there instead, every expected output must be present as a regular file, and bytecode, which nothing recompiles, is refused. -- ``install_lib`` applies the same skipped-build comparison, which covers the - legacy ``install``, ``install_lib`` and ``bdist_egg`` commands with +- ``install_lib`` applies the same skipped-build comparison to the directory + it copies from (its ``--build-dir``, by default ``build/lib``), which covers + the legacy ``install``, ``install_lib`` and ``bdist_egg`` commands with ``--skip-build``. - ``egg_info`` regenerates ``SOURCES.txt`` from the tree and never reads a leftover manifest back in, so a stale entry can add a file neither to a wheel @@ -38,10 +41,15 @@ whole directory into the archive. The directory is checked before the writers run as well as after: a writer opens each metadata file by name and would follow a symlink, overwriting whatever it points at, so the directory - itself and every entry in it must be regular. -- ``sdist`` refuses a leftover release tree (``universal_law_query-0.1.0/``, - kept by ``--keep-temp`` or an interrupted run), which setuptools would - otherwise fill, archive whole and only then remove. + itself and every entry in it must be a regular file with a single link; a + hard-linked file shares its bytes with another path and would be written + through just the same. +- ``sdist`` refuses a symlink among the files the manifest names, or a + symlinked directory above one, before copying anything: the release tree + would otherwise hold the link's target as a regular file. It also refuses a + leftover release tree (``universal_law_query-0.1.0/``, kept by + ``--keep-temp`` or an interrupted run), which setuptools would otherwise + fill, archive whole and only then remove. A symlinked directory above any of these trees (``build/`` above ``build/lib`` and the staging tree, ``src/`` above the egg-info directory) is refused before @@ -125,22 +133,29 @@ def _refuse(unexpected: list[str], tree: str) -> None: + '. Remove build leftovers before installing: ' + _remedy(tree)) -def _source_pairs(build_py_command) -> list[tuple[str, str]]: - """(source, output) for every module and data file the build copies.""" +def _under(tree: str, path: str, build_lib: str) -> str: + """``path``, produced under ``build_lib``, relocated under ``tree``.""" + return os.path.join(tree, os.path.relpath(path, build_lib)) + + +def _source_pairs(build_py_command, tree: str | None = None) -> list[tuple[str, str]]: + """(source, output) for every module and data file the build copies, with + the outputs under ``tree`` (by default the build's own ``build_lib``).""" + build_lib = build_py_command.build_lib + tree = build_lib if tree is None else tree pairs = [] for package, module, module_file in build_py_command.find_all_modules(): - output = build_py_command.get_module_outfile(build_py_command.build_lib, package.split('.'), module) - pairs.append((module_file, output)) + pairs.append((module_file, build_py_command.get_module_outfile(tree, package.split('.'), module))) for _package, src_dir, build_dir, filenames in build_py_command.data_files: for filename in filenames: - pairs.append((os.path.join(src_dir, filename), os.path.join(build_dir, filename))) + pairs.append((os.path.join(src_dir, filename), _under(tree, os.path.join(build_dir, filename), build_lib))) return pairs -def _symlinked_sources(build_py_command) -> list[str]: - """Package sources, and directories above them, that are symlinks.""" +def _symlinked_paths(paths) -> list[str]: + """Each of ``paths`` that is a symlink, and every symlinked directory above one.""" found = set() - for source, _output in _source_pairs(build_py_command): + for source in paths: path = os.path.normpath(source) while path and path != os.path.dirname(path): if os.path.islink(path): @@ -149,6 +164,11 @@ def _symlinked_sources(build_py_command) -> list[str]: return sorted(found) +def _symlinked_sources(build_py_command) -> list[str]: + """Package sources, and directories above them, that are symlinks.""" + return _symlinked_paths(source for source, _output in _source_pairs(build_py_command)) + + def _symlinked_parents(path: str) -> list[str]: """Directories above a relative ``path`` that are symlinks; a write under ``path`` would land wherever they point. An absolute path, configured @@ -179,6 +199,8 @@ def _egg_info_leftovers(tree: str, written: set[str]) -> list[str]: found.append(_relative(path) + ' (symlink)') elif not os.path.isfile(path): found.append(_relative(path) + ' (not a regular file)') + elif os.stat(path).st_nlink > 1: + found.append(_relative(path) + ' (hard link)') elif name not in written: found.append(_relative(path)) return sorted(found) @@ -187,16 +209,16 @@ def _egg_info_leftovers(tree: str, written: set[str]) -> list[str]: def _refuse_sources(symlinked: list[str]) -> None: if symlinked: raise SetupError( - 'REFUSED: symlinked package source, which would ship the link\'s target rather than a file of ' + 'REFUSED: symlinked source, which would ship the link\'s target rather than a file of ' 'this source tree: ' + ', '.join(symlinked) + '. Replace the symlink with a regular file before installing.') -def _differing_outputs(build_py_command) -> list[str]: +def _differing_outputs(build_py_command, tree: str | None = None) -> list[str]: """Produced module and data paths that are missing, not regular files, or whose bytes are not this source tree's.""" found = [] - for source, output in _source_pairs(build_py_command): + for source, output in _source_pairs(build_py_command, tree): if os.path.islink(output): continue # already reported by the tree walk if not os.path.exists(output): @@ -208,29 +230,34 @@ def _differing_outputs(build_py_command) -> list[str]: return found -def _guard_build_lib(build_py_command, compare_bytes: bool = False, recompiles: bool = True) -> None: - """Refuse a symlinked source, then anything under build/lib that is not - this source tree's output. +def _guard_build_lib(build_py_command, compare_bytes: bool = False, recompiles: bool = True, + tree: str | None = None) -> None: + """Refuse a symlinked source, then anything under the output tree (by + default build/lib) that is not this source tree's output. Bytecode is admitted only when this run rewrites it: ``recompiles`` is false when nothing is rebuilt (``--skip-build``), and setuptools skips the compile step under ``python -B`` or ``PYTHONDONTWRITEBYTECODE``. With ``compare_bytes`` each produced module's and data file's bytes are also checked against its source, and every expected output must be present. + ``tree`` names another directory holding the same layout, such as the one + ``install_lib --build-dir`` copies from. """ + build_lib = build_py_command.build_lib + tree = build_lib if tree is None else tree _refuse_sources(_symlinked_sources(build_py_command)) admit_bytecode = recompiles and not sys.dont_write_bytecode - produced = {_canonical(path) for path in build_py_command.get_outputs(include_bytecode=admit_bytecode)} - unexpected = sorted(_symlinked_parents(build_py_command.build_lib) - + _unexpected(build_py_command.build_lib, produced)) + produced = {_canonical(_under(tree, path, build_lib)) + for path in build_py_command.get_outputs(include_bytecode=admit_bytecode)} + unexpected = sorted(_symlinked_parents(tree) + _unexpected(tree, produced)) if compare_bytes: - unexpected = sorted(unexpected + _differing_outputs(build_py_command)) - _refuse(unexpected, build_py_command.build_lib) + unexpected = sorted(unexpected + _differing_outputs(build_py_command, tree)) + _refuse(unexpected, tree) -def _guard_skipped_build(command) -> None: - """A command that ships build/lib without rebuilding it compares the bytes it has.""" - _guard_build_lib(command.get_finalized_command('build_py'), compare_bytes=True, recompiles=False) +def _guard_skipped_build(command, tree: str | None = None) -> None: + """A command that ships a build tree without rebuilding it compares the bytes it has.""" + _guard_build_lib(command.get_finalized_command('build_py'), compare_bytes=True, recompiles=False, tree=tree) class build_py(_build_py): @@ -242,7 +269,9 @@ def run(self) -> None: # its source would otherwise be kept as "up to date" and shipped. self.force = True super().run() - _guard_build_lib(self, compare_bytes=True) + # A dry run writes nothing, so there is nothing to compare; the + # leftover check still runs. + _guard_build_lib(self, compare_bytes=not self.dry_run) class bdist_wheel(_bdist_wheel): @@ -264,7 +293,8 @@ class install_lib(_install_lib): def run(self) -> None: if self.skip_build: - _guard_skipped_build(self) + # The directory this command copies from, not necessarily build/lib. + _guard_skipped_build(self, self.build_dir) super().run() @@ -304,6 +334,9 @@ class sdist(_sdist): """Refuse a leftover release tree; setuptools would fill it, archive it whole and only then remove it.""" def make_release_tree(self, base_dir, files) -> None: + # Every manifest entry is copied or hard-linked into the tree and then + # archived, so a symlink among them, or above one, would ship its target. + _refuse_sources(_symlinked_paths(files)) _refuse(_unexpected(base_dir, set()), base_dir) super().make_release_tree(base_dir, files) diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index 6e3e3b1..f89f581 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -525,7 +525,7 @@ def test_symlinked_package_sources_are_refused(self): result = self.wheel() output = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, 'a symlinked source must be refused') - self.assertIn('REFUSED: symlinked package source', output) + self.assertIn('REFUSED: symlinked source', output) self.assertIn('src/' + PACKAGE + '/leak.py (symlink)', output) self.assertEqual(sorted(self.wheels.glob('*.whl')), []) self.assertFalse((self.checkout / 'build' / 'lib' / PACKAGE / 'leak.py').exists(), @@ -557,10 +557,93 @@ def test_symlinked_package_sources_are_refused(self): result = direct_setup(self.checkout, 'build_py') output = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, 'a symlinked package directory must be refused') - self.assertIn('REFUSED: symlinked package source', output) + self.assertIn('REFUSED: symlinked source', output) self.assertIn('src/' + PACKAGE + '/sub (symlink)', output) self.assertFalse((self.checkout / 'build' / 'lib' / PACKAGE / 'sub').exists()) + def test_symlinked_sdist_inputs_are_refused(self): + # The manifest names more than the package sources (tests, docs, the + # wrappers); each entry is copied or hard-linked into the release tree + # and archived, so a symlink among them would ship its target. + private = self.base / 'private_test.py' + private.write_text('PRIVATE = True\n', encoding='utf-8') + leak = self.checkout / 'tests' / 'test_leak.py' + try: + os.symlink(private, leak) + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'sdist', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked sdist input must be refused') + self.assertIn('REFUSED: symlinked source', output) + self.assertIn('tests/test_leak.py (symlink)', output) + self.assertEqual(sorted(dist.glob('*.tar.gz')) if dist.exists() else [], []) + self.assertFalse((self.checkout / release_tree_name(self.checkout)).exists(), + 'nothing may be copied before the refusal') + self.assertTrue(leak.is_symlink(), 'the link is reported, not replaced') + + def test_hard_linked_egg_info_entries_are_refused_before_anything_is_written(self): + # A hard link is a regular file by every test but shares its bytes with + # another path, so a writer opening it by name overwrites that path too. + outside = self.base / 'outside' + outside.mkdir() + secret = outside / 'secret.txt' + secret.write_text('SECRET\n', encoding='utf-8') + egg_info = self.checkout / 'src' / 'universal_law_query.egg-info' + egg_info.mkdir() + try: + os.link(secret, egg_info / 'SOURCES.txt') + except (OSError, NotImplementedError) as error: + self.skipTest(f'hard links are not available here: {error}') + result = direct_setup(self.checkout, 'egg_info') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a hard-linked metadata file must be refused') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('src/universal_law_query.egg-info/SOURCES.txt (hard link)', output) + self.assertEqual(secret.read_text(encoding='utf-8'), 'SECRET\n', 'nothing may be written through the link') + self.assertFalse((egg_info / 'PKG-INFO').exists(), 'no writer may run before the refusal') + result = direct_setup(self.checkout, 'sdist', '--dist-dir', str(self.base / 'dist')) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(secret.read_text(encoding='utf-8'), 'SECRET\n') + + def test_dry_run_build_is_not_refused_for_writing_nothing(self): + # ``--dry-run`` writes nothing by design, so the post-run comparison + # has nothing to compare and is skipped; the leftover check still runs. + result = direct_setup(self.checkout, '--dry-run', 'build_py') + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, output) + self.assertNotIn('REFUSED', output) + self.assertFalse((self.checkout / 'build').exists(), 'a dry run must write nothing') + plant_leftovers(self.checkout) + result = direct_setup(self.checkout, '--dry-run', 'build_py') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'leftovers are refused on a dry run as well') + self.assertIn('REFUSED: stale build output', output) + + def test_install_lib_skip_build_validates_the_directory_it_copies_from(self): + # ``install_lib --build-dir`` copies from the named directory, so that + # directory, not the default build/lib, is what must hold exactly this + # tree's outputs. + built = direct_setup(self.checkout, 'build_py') + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + elsewhere = self.checkout / 'elsewhere' + (elsewhere / PACKAGE).mkdir(parents=True) + (elsewhere / PACKAGE / 'stale.py').write_text('STALE = True\n', encoding='utf-8') + result = direct_setup(self.checkout, 'install_lib', '--skip-build', '--build-dir', 'elsewhere', + '--install-dir', str(self.target)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'the copied-from directory must be validated') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('elsewhere/' + PACKAGE + '/stale.py', output) + self.assertIn('elsewhere/' + PACKAGE + '/__init__.py (missing)', output) + self.assertIn('under elsewhere', output) + self.assertEqual(installed_files(self.target), [], 'nothing may be installed after a refusal') + result = direct_setup(self.checkout, 'install_lib', '--skip-build', '--build-dir', 'build/lib', + '--install-dir', str(self.target)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(installed_files(self.target), tracked_source_files()) + def test_symlinked_egg_info_entries_are_refused_before_anything_is_written(self): # egg_info's writers open each metadata file by name and follow a # symlink, so a link planted under a writer's name would have its From 3a38d51e44d2f25fab3681763788e6d712a89fb6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 12:10:59 +0000 Subject: [PATCH 08/14] Refuse hard-linked build outputs and a leftover bdist_egg staging tree Answers the automatic review at 83019d82 (two findings): - A hard-linked file under build/lib is refused like a symlink: its bytes are shared with another path, so it is not this build's output. The write-through the finding describes does not reproduce on setuptools 77.0.3 (the copy unlinks the destination before writing, and the other link keeps its bytes), but the rule should not depend on that detail. - bdist_egg refuses a leftover staging tree under build/bdist.*/egg the same way bdist_wheel does; the legacy archive is assembled from whatever that tree holds. Controls: test_hard_linked_build_outputs_are_refused (the other link keeps its bytes and its link count) and test_stale_bdist_egg_staging_tree_is_refused (refused, then a clean egg holds exactly the tracked files); both red at 16fb1e64 and green here. Provenance 30 OK (also under -O), discovery 119 OK, legacy 38 OK. README updated. Scientific effect: NONE. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 16 ++++++---- setup.py | 24 ++++++++++++--- tests/test_install_provenance.py | 50 ++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e2deb07..79bf2ee 100644 --- a/README.md +++ b/README.md @@ -70,12 +70,16 @@ stale `SOURCES.txt` is never read back, so it can add a file neither to a wheel nor to a `setup.py sdist` archive; any other file left in the egg-info directory is refused, because an sdist and a direct `bdist_wheel` would copy it into the archive, and so is a leftover sdist release tree -(`universal_law_query-0.1.0/`), which an sdist would archive whole, and a -symlink among the files an sdist's manifest names. A symlink or hard link in -or at the egg-info directory, or a symlinked `build/`, is refused before -anything is written, because setuptools' metadata writers and build copies -would otherwise overwrite whatever the link points at. A `--dry-run` build -writes nothing and is not refused for that; leftovers are still refused. +(`universal_law_query-0.1.0/`), which an sdist would archive whole, a +symlink among the files an sdist's manifest names, and a leftover +`bdist_egg` staging tree under `build/bdist.*/egg/`. A symlink or hard link +in or at the egg-info directory, a hard link under `build/lib/`, or a +symlinked `build/`, is refused before anything is written, because +setuptools' metadata writers and build copies would otherwise overwrite, or +share bytes with, whatever the link points at (an sdist kept with +`--keep-temp` hard-links the egg-info files into its release tree, which is +itself refused; remove that tree first). A `--dry-run` build writes nothing +and is not refused for that; leftovers are still refused. `pyproject.toml` also declares that the package ships no package data. Nothing is deleted for you; remove the leftovers before installing: `git clean -fdX -- build src/universal_law_query.egg-info`. diff --git a/setup.py b/setup.py index ffadfef..123e9b9 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools reuses ``build/`` between builds and removes nothing from it. A module left under ``build/lib`` by an install from another commit is copied into the next wheel although ``src/`` does not contain it, and an import check -of the installed package still passes. Five commands are guarded: +of the installed package still passes. Six commands are guarded: - ``build_py`` refuses a symlink among the package sources it would copy, or a symlinked directory above one, because the copy and the byte comparison @@ -15,7 +15,8 @@ before copying anything and again afterwards, and refuses when anything else is present. Every symlink under the tree is refused wherever it points, because the walk does not follow symlinks but the wheel staging step - dereferences them. The run rewrites every file it produces (setuptools would + dereferences them; a hard-linked file is refused as well, since its bytes + are shared with another path. The run rewrites every file it produces (setuptools would otherwise skip a destination that is newer than its source), and afterwards each produced module's bytes are compared with its source, so bytes left at a path this run writes are replaced, never shipped. Bytecode counts as produced @@ -34,6 +35,9 @@ it copies from (its ``--build-dir``, by default ``build/lib``), which covers the legacy ``install``, ``install_lib`` and ``bdist_egg`` commands with ``--skip-build``. +- ``bdist_egg`` refuses a leftover staging tree under ``build/bdist.*/egg`` + the same way ``bdist_wheel`` does; the legacy archive is assembled from + whatever that tree holds. - ``egg_info`` regenerates ``SOURCES.txt`` from the tree and never reads a leftover manifest back in, so a stale entry can add a file neither to a wheel nor to a source distribution; any other file left beside the ones this run @@ -68,6 +72,7 @@ from importlib import metadata as _metadata from setuptools import setup +from setuptools.command.bdist_egg import bdist_egg as _bdist_egg from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools.command.build_py import build_py as _build_py from setuptools.command.egg_info import egg_info as _egg_info @@ -110,6 +115,8 @@ def _unexpected(tree: str, produced: set[str], empty_dirs: bool = False) -> list path = os.path.join(dirpath, name) if os.path.islink(path): found.append(_relative(path) + ' (symlink)') + elif os.stat(path).st_nlink > 1: + found.append(_relative(path) + ' (hard link)') elif _canonical(path) not in produced: found.append(_relative(path)) return sorted(found) @@ -288,6 +295,15 @@ def run(self) -> None: super().run() +class bdist_egg(_bdist_egg): + """Refuse a leftover egg staging tree; the legacy archive is assembled from whatever it holds.""" + + def run(self) -> None: + _refuse(sorted(_symlinked_parents(self.bdist_dir) + _unexpected(self.bdist_dir, set(), empty_dirs=True)), + self.bdist_dir) + super().run() + + class install_lib(_install_lib): """The legacy install, install_lib and bdist_egg paths with --skip-build copy build/lib as is.""" @@ -341,5 +357,5 @@ def make_release_tree(self, base_dir, files) -> None: super().make_release_tree(base_dir, files) -setup(cmdclass={'build_py': build_py, 'bdist_wheel': bdist_wheel, 'install_lib': install_lib, - 'egg_info': egg_info, 'sdist': sdist}) +setup(cmdclass={'build_py': build_py, 'bdist_wheel': bdist_wheel, 'bdist_egg': bdist_egg, + 'install_lib': install_lib, 'egg_info': egg_info, 'sdist': sdist}) diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index f89f581..8378228 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -108,6 +108,11 @@ def wheel_staging_dir(checkout: Path) -> Path: return checkout / raw.strip() +def egg_staging_dir(checkout: Path) -> Path: + # bdist_egg stages under the same bdist base as bdist_wheel (``build/bdist./egg``). + return wheel_staging_dir(checkout).parent / 'egg' + + def plant_stale_manifest(checkout: Path) -> Path: # A SOURCES.txt left by an earlier build, naming an untracked file that exists. egg_info = checkout / 'src' / 'universal_law_query.egg-info' @@ -269,6 +274,51 @@ def test_stale_wheel_staging_tree_is_refused(self): self.assertIn(staged.parent.relative_to(self.checkout).as_posix() + ' (empty directory)', output) self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + def test_stale_bdist_egg_staging_tree_is_refused(self): + # The legacy egg is assembled from build/bdist./egg, which a + # --keep-temp or interrupted run leaves behind with whatever it held. + staged = egg_staging_dir(self.checkout) / PACKAGE / 'stage_stale.py' + staged.parent.mkdir(parents=True) + staged.write_text('STAGE_STALE = True\n', encoding='utf-8') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_egg', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'the egg build must fail closed on a staging leftover') + self.assertIn('REFUSED: stale build output', output) + self.assertIn(staged.relative_to(self.checkout).as_posix(), output) + self.assertIn('git clean -fdX -- build src/universal_law_query.egg-info', output) + self.assertEqual(sorted(dist.glob('*.egg')) if dist.exists() else [], []) + self.assertTrue(staged.is_file(), 'the leftover is reported, not deleted') + # A clean egg build holds exactly the tracked package files. + shutil.rmtree(egg_staging_dir(self.checkout).parent) + result = direct_setup(self.checkout, 'bdist_egg', '--dist-dir', str(dist)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + eggs = sorted(dist.glob('*.egg')) + self.assertEqual(len(eggs), 1, eggs) + self.assertEqual(wheel_package_files(eggs[0]), tracked_source_files()) + + def test_hard_linked_build_outputs_are_refused(self): + # A hard link at a produced path shares its bytes with another file. + # setuptools' copy unlinks the destination before writing, so the other + # file is not written through, but the tree is still not this build's + # output; like a symlink it is refused before anything is copied. + outside = self.base / 'outside.py' + outside.write_text('OUTSIDE = True\n', encoding='utf-8') + linked = self.checkout / 'build' / 'lib' / PACKAGE / 'catalog.py' + linked.parent.mkdir(parents=True) + try: + os.link(outside, linked) + except (OSError, NotImplementedError) as error: + self.skipTest(f'hard links are not available here: {error}') + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a hard-linked build output must be refused') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('build/lib/' + PACKAGE + '/catalog.py (hard link)', output) + self.assertEqual(outside.read_text(encoding='utf-8'), 'OUTSIDE = True\n') + self.assertEqual(os.stat(outside).st_nlink, 2, 'the link is reported, not removed') + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + def test_direct_bdist_wheel_skip_build_still_refuses_stale_build_lib(self): # ``--skip-build`` never runs build_py, so the wheel command must repeat # the inventory check itself. From 7191e599d54d53461810813028bfd022e262b336 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 15:18:34 +0000 Subject: [PATCH 09/14] Validate install_lib's source on every path, a pre-generated dist-info, the shared bdist base and Windows junctions Answers the automatic review at 3a38d51e (four findings) and the two README inaccuracies the nonauthor read at that head named: - install_lib validates the directory it copies from (--build-dir) before every copy, not only under --skip-build; bytecode a build in the same run wrote is admitted, and a dry run copies nothing. - bdist_wheel refuses a pre-generated --dist-info-dir holding anything its own metadata conversion would not produce: an alias, a hard link, a subdirectory other than licenses/, an unknown file, or a license file the distribution does not declare. - Every bdist command (bdist_wheel, bdist_egg, bdist_dumb, bdist_rpm) refuses a leftover anywhere under the shared build/bdist. base and under its own staging directory before anything is assembled. bdist_dumb and bdist_rpm are imported through distutils.command, the name setuptools itself patches, so no second copy of those modules is loaded. - A Windows junction counts as a symlink everywhere one is refused: os.path.isjunction on Python 3.12+, otherwise the mount-point reparse tag on os.lstat. - README: setuptools copies the sdist release tree, it does not hard-link the egg-info files into it, and the build copy does not write through a hard link under build/lib; a hard link there is refused because the tree is not this build's output. Controls: test_pre_generated_dist_info_is_validated, test_stale_bdist_dumb_and_rpm_staging_trees_are_refused, test_windows_junctions_are_treated_as_symlinks (both detection routes simulated, since this host has no junctions), and an extended test_install_lib_skip_build_validates_the_directory_it_copies_from; all four red at 3a38d51e, green here. Provenance 33 OK (also under -O), discovery 122 OK, legacy 38 OK, isolated opt-in OK. Scientific effect: NONE. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 32 ++--- setup.py | 205 ++++++++++++++++++++++++------- tests/test_install_provenance.py | 135 ++++++++++++++++++++ 3 files changed, 315 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 79bf2ee..d4ef8eb 100644 --- a/README.md +++ b/README.md @@ -60,26 +60,30 @@ among the package sources in `src/`, because the copy would ship its target. Bytecode under `build/lib/` is accepted only from the run that compiles it (`build_py --compile` with bytecode writing enabled); a cache file found by a run that does not recompile it, such as `--skip-build` or a build under -`python -B`, is refused. The wheel command refuses a leftover staging tree -under `build/bdist.*/wheel/` (an interrupted build leaves one) and repeats the +`python -B`, is refused. Every `bdist` command (`bdist_wheel`, `bdist_egg`, +`bdist_dumb`, `bdist_rpm`) refuses a leftover anywhere under the shared +`build/bdist.*/` base (an interrupted build leaves one), the wheel command +also refuses a pre-generated `--dist-info-dir` holding anything its own +metadata conversion would not produce, and the wheel command repeats the `build/lib/` check, so `--skip-build` cannot bypass it; with `--skip-build` every expected output must be present with the bytes of its source. The -legacy `install`, `install_lib` and `bdist_egg` commands with `--skip-build` -run the same check. The egg-info manifest is regenerated from the tree and a +legacy `install`, `install_lib` and `bdist_egg` commands run the same check +on the directory `install_lib` copies from (`--build-dir`), with or without +`--skip-build`. The egg-info manifest is regenerated from the tree and a stale `SOURCES.txt` is never read back, so it can add a file neither to a wheel nor to a `setup.py sdist` archive; any other file left in the egg-info directory is refused, because an sdist and a direct `bdist_wheel` would copy it into the archive, and so is a leftover sdist release tree -(`universal_law_query-0.1.0/`), which an sdist would archive whole, a -symlink among the files an sdist's manifest names, and a leftover -`bdist_egg` staging tree under `build/bdist.*/egg/`. A symlink or hard link -in or at the egg-info directory, a hard link under `build/lib/`, or a -symlinked `build/`, is refused before anything is written, because -setuptools' metadata writers and build copies would otherwise overwrite, or -share bytes with, whatever the link points at (an sdist kept with -`--keep-temp` hard-links the egg-info files into its release tree, which is -itself refused; remove that tree first). A `--dry-run` build writes nothing -and is not refused for that; leftovers are still refused. +(`universal_law_query-0.1.0/`), which an sdist would archive whole, and a +symlink among the files an sdist's manifest names. A symlink or hard link +in or at the egg-info directory, or a symlinked `build/`, is refused before +anything is written, because setuptools' metadata writers and build copies +would otherwise write through to whatever the link points at. A hard link +under `build/lib/` is refused because its bytes are shared with another path, +so the tree is not this build's output; the build copy itself unlinks its +destination before writing and does not write through. A Windows junction +counts as a symlink throughout. A `--dry-run` build writes nothing and is not +refused for that; leftovers are still refused. `pyproject.toml` also declares that the package ships no package data. Nothing is deleted for you; remove the leftovers before installing: `git clean -fdX -- build src/universal_law_query.egg-info`. diff --git a/setup.py b/setup.py index 123e9b9..3edd8a5 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools reuses ``build/`` between builds and removes nothing from it. A module left under ``build/lib`` by an install from another commit is copied into the next wheel although ``src/`` does not contain it, and an import check -of the installed package still passes. Six commands are guarded: +of the installed package still passes. Eight commands are guarded: - ``build_py`` refuses a symlink among the package sources it would copy, or a symlinked directory above one, because the copy and the byte comparison @@ -15,8 +15,9 @@ before copying anything and again afterwards, and refuses when anything else is present. Every symlink under the tree is refused wherever it points, because the walk does not follow symlinks but the wheel staging step - dereferences them; a hard-linked file is refused as well, since its bytes - are shared with another path. The run rewrites every file it produces (setuptools would + dereferences them (a Windows junction counts as a symlink throughout); a + hard-linked file is refused as well, since its bytes are shared with another + path. The run rewrites every file it produces (setuptools would otherwise skip a destination that is newer than its source), and afterwards each produced module's bytes are compared with its source, so bytes left at a path this run writes are replaced, never shipped. Bytecode counts as produced @@ -25,19 +26,23 @@ skips the compile step, so a cache file found under ``build/lib`` is refused. A dry run writes nothing, so the comparison after the run is skipped there; the leftover check before it still applies. -- ``bdist_wheel`` refuses a leftover staging tree under ``build/bdist.*/wheel`` - (an interrupted build leaves one, and the next build archives whatever it - holds) and repeats the ``build/lib`` comparison, so ``--skip-build`` cannot +- ``bdist_wheel`` refuses a leftover anywhere under the shared bdist base + (``build/bdist.``, where the wheel, egg, dumb and rpm staging + trees live; an interrupted build leaves one, and the next build archives + whatever it holds) and under its own staging directory, refuses a + pre-generated ``--dist-info-dir`` holding anything its own metadata + conversion would not produce (the directory is copied into the wheel as + is), and repeats the ``build/lib`` comparison, so ``--skip-build`` cannot bypass it. With ``--skip-build`` nothing is rebuilt, so the byte comparison runs there instead, every expected output must be present as a regular file, and bytecode, which nothing recompiles, is refused. -- ``install_lib`` applies the same skipped-build comparison to the directory - it copies from (its ``--build-dir``, by default ``build/lib``), which covers - the legacy ``install``, ``install_lib`` and ``bdist_egg`` commands with - ``--skip-build``. -- ``bdist_egg`` refuses a leftover staging tree under ``build/bdist.*/egg`` - the same way ``bdist_wheel`` does; the legacy archive is assembled from - whatever that tree holds. +- ``install_lib`` applies the same comparison to the directory it copies from + (its ``--build-dir``, by default ``build/lib``), whether or not it rebuilt + it, which covers the legacy ``install``, ``install_lib`` and ``bdist_egg`` + commands with and without ``--skip-build``. +- ``bdist_egg``, ``bdist_dumb`` and ``bdist_rpm`` refuse a leftover under the + shared bdist base and their own staging trees the same way; the legacy + archives are assembled from whatever those trees hold. - ``egg_info`` regenerates ``SOURCES.txt`` from the tree and never reads a leftover manifest back in, so a stale entry can add a file neither to a wheel nor to a source distribution; any other file left beside the ones this run @@ -68,10 +73,17 @@ import filecmp import os +import stat import sys from importlib import metadata as _metadata from setuptools import setup +# The two legacy commands live in distutils; import them through the name +# setuptools itself uses. Importing ``setuptools._distutils.…`` directly would +# load second copies of those modules and rebind the shared package's +# attributes to unpatched classes. +from distutils.command.bdist_dumb import bdist_dumb as _bdist_dumb # noqa: I001 +from distutils.command.bdist_rpm import bdist_rpm as _bdist_rpm from setuptools.command.bdist_egg import bdist_egg as _bdist_egg from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools.command.build_py import build_py as _build_py @@ -82,6 +94,9 @@ from setuptools.errors import SetupError EGG_INFO = 'src/universal_law_query.egg-info' +# Windows' reparse tag for a junction (mount point). The ``stat`` module only +# defines the name on Windows builds; the value is fixed by the platform. +_IO_REPARSE_TAG_MOUNT_POINT = getattr(stat, 'IO_REPARSE_TAG_MOUNT_POINT', 0xA0000003) REMEDY = 'git clean -fdX -- build ' + EGG_INFO @@ -97,24 +112,47 @@ def _relative(path: str) -> str: return relative.replace(os.sep, '/') +def _alias(path: str) -> str | None: + """``'symlink'`` or ``'junction'`` when ``path`` points somewhere else, else None. + + A Windows junction is a directory reparse point that ``os.path.islink`` + does not report; Python 3.12 has ``os.path.isjunction``, and older + interpreters expose the reparse tag on ``os.lstat``. + """ + if os.path.islink(path): + return 'symlink' + isjunction = getattr(os.path, 'isjunction', None) + if isjunction is not None: + return 'junction' if isjunction(path) else None + try: + tag = getattr(os.lstat(path), 'st_reparse_tag', 0) + except OSError: + return None + return 'junction' if tag == _IO_REPARSE_TAG_MOUNT_POINT else None + + def _unexpected(tree: str, produced: set[str], empty_dirs: bool = False) -> list[str]: - """Files under ``tree`` that are not in ``produced``, every symlink, and - with ``empty_dirs`` every empty directory below the tree itself.""" + """Files under ``tree`` that are not in ``produced``, every symlink or + junction, every hard-linked file, and with ``empty_dirs`` every empty + directory below the tree itself.""" found = [] - if os.path.islink(tree): - return [_relative(tree) + ' (symlink)'] + kind = _alias(tree) + if kind: + return [_relative(tree) + ' (' + kind + ')'] if not os.path.isdir(tree): return found for dirpath, dirnames, filenames in os.walk(tree): if empty_dirs and dirpath != tree and not dirnames and not filenames: found.append(_relative(dirpath) + ' (empty directory)') for name in dirnames: - if os.path.islink(os.path.join(dirpath, name)): - found.append(_relative(os.path.join(dirpath, name)) + ' (symlink)') + kind = _alias(os.path.join(dirpath, name)) + if kind: + found.append(_relative(os.path.join(dirpath, name)) + ' (' + kind + ')') for name in filenames: path = os.path.join(dirpath, name) - if os.path.islink(path): - found.append(_relative(path) + ' (symlink)') + kind = _alias(path) + if kind: + found.append(_relative(path) + ' (' + kind + ')') elif os.stat(path).st_nlink > 1: found.append(_relative(path) + ' (hard link)') elif _canonical(path) not in produced: @@ -160,13 +198,14 @@ def _source_pairs(build_py_command, tree: str | None = None) -> list[tuple[str, def _symlinked_paths(paths) -> list[str]: - """Each of ``paths`` that is a symlink, and every symlinked directory above one.""" + """Each of ``paths`` that is a symlink or junction, and every such directory above one.""" found = set() for source in paths: path = os.path.normpath(source) while path and path != os.path.dirname(path): - if os.path.islink(path): - found.add(_relative(path) + ' (symlink)') + kind = _alias(path) + if kind: + found.add(_relative(path) + ' (' + kind + ')') path = os.path.dirname(path) return sorted(found) @@ -185,8 +224,9 @@ def _symlinked_parents(path: str) -> list[str]: return found parent = os.path.dirname(os.path.normpath(path)) while parent and parent != os.path.dirname(parent): - if os.path.islink(parent): - found.append(_relative(parent) + ' (symlink)') + kind = _alias(parent) + if kind: + found.append(_relative(parent) + ' (' + kind + ')') parent = os.path.dirname(parent) return found @@ -196,14 +236,16 @@ def _egg_info_leftovers(tree: str, written: set[str]) -> list[str]: name this run writes: a symlink anywhere (a writer would follow it and overwrite its target), a directory, or a file no writer produces.""" found = _symlinked_parents(tree) - if os.path.islink(tree): - return sorted(found + [_relative(tree) + ' (symlink)']) + kind = _alias(tree) + if kind: + return sorted(found + [_relative(tree) + ' (' + kind + ')']) if not os.path.isdir(tree): return sorted(found) for name in sorted(os.listdir(tree)): path = os.path.join(tree, name) - if os.path.islink(path): - found.append(_relative(path) + ' (symlink)') + kind = _alias(path) + if kind: + found.append(_relative(path) + ' (' + kind + ')') elif not os.path.isfile(path): found.append(_relative(path) + ' (not a regular file)') elif os.stat(path).st_nlink > 1: @@ -226,7 +268,7 @@ def _differing_outputs(build_py_command, tree: str | None = None) -> list[str]: whose bytes are not this source tree's.""" found = [] for source, output in _source_pairs(build_py_command, tree): - if os.path.islink(output): + if _alias(output): continue # already reported by the tree walk if not os.path.exists(output): found.append(_relative(output) + ' (missing)') @@ -267,6 +309,62 @@ def _guard_skipped_build(command, tree: str | None = None) -> None: _guard_build_lib(command.get_finalized_command('build_py'), compare_bytes=True, recompiles=False, tree=tree) +def _dist_info_leftovers(tree: str, distribution) -> list[str]: + """Entries of a pre-generated dist-info directory that this build's own + metadata conversion would not produce, plus every alias or hard link. + + ``bdist_wheel --dist-info-dir`` copies the directory into the wheel as is. + The conversion keeps the egg-info writers' files except ``PKG-INFO`` (which + becomes ``METADATA``), ``requires.txt`` and ``SOURCES.txt``, and places the + distribution's license files under ``licenses/``. + """ + written = {entry.name for entry in _metadata.entry_points(group='egg_info.writers')} + written -= {'PKG-INFO', 'requires.txt', 'SOURCES.txt'} + written |= {'METADATA', 'WHEEL', 'RECORD'} + licenses = {os.path.normpath(path).replace(os.sep, '/') + for path in (distribution.metadata.license_files or ())} + found = _symlinked_parents(tree) + kind = _alias(tree) + if kind: + return sorted(found + [_relative(tree) + ' (' + kind + ')']) + if not os.path.isdir(tree): + return sorted(found + [_relative(tree) + ' (missing)']) + for dirpath, dirnames, filenames in os.walk(tree): + inside = os.path.relpath(dirpath, tree).replace(os.sep, '/') + for name in dirnames: + path = os.path.join(dirpath, name) + kind = _alias(path) + if kind: + found.append(_relative(path) + ' (' + kind + ')') + elif not (inside == os.curdir and name == 'licenses'): + found.append(_relative(path) + ' (directory)') + for name in filenames: + path = os.path.join(dirpath, name) + kind = _alias(path) + if kind: + found.append(_relative(path) + ' (' + kind + ')') + elif os.stat(path).st_nlink > 1: + found.append(_relative(path) + ' (hard link)') + elif inside == os.curdir: + if name not in written: + found.append(_relative(path)) + elif inside == 'licenses': + if name not in licenses: + found.append(_relative(path)) + else: + found.append(_relative(path)) + return sorted(found) + + +def _guard_staging(command, staging_dir: str) -> None: + """Refuse any leftover under the shared bdist base (``build/bdist.``), + where the wheel, egg, dumb and rpm staging trees live, and under the + command's own staging directory when it was configured elsewhere.""" + base = command.get_finalized_command('bdist').bdist_base + for tree in (base, staging_dir): + _refuse(sorted(_symlinked_parents(tree) + _unexpected(tree, set(), empty_dirs=True)), tree) + + class build_py(_build_py): """Refuse build output that this source tree did not produce.""" @@ -285,8 +383,11 @@ class bdist_wheel(_bdist_wheel): """Refuse a leftover staging tree, and re-check build/lib even with --skip-build.""" def run(self) -> None: - _refuse(sorted(_symlinked_parents(self.bdist_dir) + _unexpected(self.bdist_dir, set(), empty_dirs=True)), - self.bdist_dir) + _guard_staging(self, self.bdist_dir) + if self.dist_info_dir: + # Pre-generated metadata is copied into the wheel as is, so it must + # hold only what this build's own conversion would produce. + _refuse(_dist_info_leftovers(self.dist_info_dir, self.distribution), self.dist_info_dir) if self.skip_build: _guard_skipped_build(self) else: @@ -299,21 +400,37 @@ class bdist_egg(_bdist_egg): """Refuse a leftover egg staging tree; the legacy archive is assembled from whatever it holds.""" def run(self) -> None: - _refuse(sorted(_symlinked_parents(self.bdist_dir) + _unexpected(self.bdist_dir, set(), empty_dirs=True)), - self.bdist_dir) + _guard_staging(self, self.bdist_dir) super().run() -class install_lib(_install_lib): - """The legacy install, install_lib and bdist_egg paths with --skip-build copy build/lib as is.""" +class bdist_dumb(_bdist_dumb): + """Refuse a leftover dumb staging tree; the archive is the pseudo-installation tree as found.""" def run(self) -> None: - if self.skip_build: - # The directory this command copies from, not necessarily build/lib. - _guard_skipped_build(self, self.build_dir) + _guard_staging(self, self.bdist_dir) super().run() +class bdist_rpm(_bdist_rpm): + """Refuse a leftover rpm build tree under the bdist base before rpmbuild sees it.""" + + def run(self) -> None: + _guard_staging(self, self.rpm_base) + super().run() + + +class install_lib(_install_lib): + """Validate the directory this command copies from (--build-dir, by default build/lib).""" + + def install(self): + # Runs after build(): bytecode a build in this run wrote is admitted; + # under --skip-build nothing recompiles it. A dry run copies nothing. + _guard_build_lib(self.get_finalized_command('build_py'), compare_bytes=not self.dry_run, + recompiles=not self.skip_build, tree=self.build_dir) + return super().install() + + class _fresh_manifest_maker(_manifest_maker): """Build SOURCES.txt from the tree; a leftover manifest is never read back.""" @@ -350,12 +467,14 @@ class sdist(_sdist): """Refuse a leftover release tree; setuptools would fill it, archive it whole and only then remove it.""" def make_release_tree(self, base_dir, files) -> None: - # Every manifest entry is copied or hard-linked into the tree and then - # archived, so a symlink among them, or above one, would ship its target. + # Every manifest entry is copied into the tree (setuptools disables hard + # links here) and then archived, so a symlink among them, or above one, + # would ship its target as a regular file. _refuse_sources(_symlinked_paths(files)) _refuse(_unexpected(base_dir, set()), base_dir) super().make_release_tree(base_dir, files) setup(cmdclass={'build_py': build_py, 'bdist_wheel': bdist_wheel, 'bdist_egg': bdist_egg, - 'install_lib': install_lib, 'egg_info': egg_info, 'sdist': sdist}) + 'bdist_dumb': bdist_dumb, 'bdist_rpm': bdist_rpm, 'install_lib': install_lib, + 'egg_info': egg_info, 'sdist': sdist}) diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index 8378228..8be6ad7 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -96,6 +96,17 @@ def direct_setup(checkout: Path, *args: str) -> subprocess.CompletedProcess: capture_output=True, text=True, timeout=PIP_TIMEOUT) +def setup_with_preamble(checkout: Path, preamble: str, *args: str) -> subprocess.CompletedProcess: + # Runs setup.py in-process after ``preamble``, for conditions this host + # cannot create (a Windows junction), simulated where setup.py observes them. + code = (preamble + '\nimport runpy, sys\nsys.argv = ' + repr(['setup.py', *args]) + + "\nrunpy.run_path('setup.py', run_name='__main__')\n") + env = dict(os.environ) + env['PYTHONDONTWRITEBYTECODE'] = '1' + return subprocess.run([sys.executable, '-c', code], cwd=checkout, env=env, + capture_output=True, text=True, timeout=PIP_TIMEOUT) + + def wheel_staging_dir(checkout: Path) -> Path: # The directory bdist_wheel stages the archive in (``build/bdist./wheel``), # asked of setuptools itself so the platform tag matches the build's. @@ -297,6 +308,122 @@ def test_stale_bdist_egg_staging_tree_is_refused(self): self.assertEqual(len(eggs), 1, eggs) self.assertEqual(wheel_package_files(eggs[0]), tracked_source_files()) + def test_stale_bdist_dumb_and_rpm_staging_trees_are_refused(self): + # Every bdist command stages under build/bdist.; the dumb + # archive is that pseudo-installation tree as found, and rpmbuild works + # from the rpm tree, so a leftover under the shared base is refused by + # each of them before anything is assembled. + base = wheel_staging_dir(self.checkout).parent + staged = base / 'dumb' / 'private' / 'stale.txt' + staged.parent.mkdir(parents=True) + staged.write_text('STALE\n', encoding='utf-8') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_dumb', '--format', 'zip', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'the dumb build must fail closed on a staging leftover') + self.assertIn('REFUSED: stale build output', output) + self.assertIn(staged.relative_to(self.checkout).as_posix(), output) + self.assertEqual(sorted(dist.glob('*')) if dist.exists() else [], []) + self.assertTrue(staged.is_file(), 'the leftover is reported, not deleted') + # The same leftover blocks the other bdist commands, which share the base. + result = direct_setup(self.checkout, 'bdist_rpm', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0) + self.assertIn(staged.relative_to(self.checkout).as_posix(), output) + self.assertNotIn('rpmbuild', output, 'the refusal must come before rpmbuild is involved') + result = self.wheel() + self.assertNotEqual(result.returncode, 0) + self.assertIn(staged.relative_to(self.checkout).as_posix(), result.stdout + result.stderr) + # A clean dumb build is not refused. + shutil.rmtree(base) + result = direct_setup(self.checkout, 'bdist_dumb', '--format', 'zip', '--dist-dir', str(dist)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(sorted(dist.glob('*.zip'))), 1) + + def test_windows_junctions_are_treated_as_symlinks(self): + # A Windows junction is not reported by os.path.islink; Python 3.12+ + # reports it through os.path.isjunction and every supported version + # through the reparse tag on os.lstat. Neither exists here, so both are + # simulated for one real directory: an empty build/lib/ that + # a junction to an outside directory would look like, and that build_py + # would otherwise write every module into. + junction = self.checkout / 'build' / 'lib' / PACKAGE + junction.mkdir(parents=True) + target = os.path.normcase(os.path.abspath(junction)) + via_isjunction = ('import os\n_t = ' + repr(target) + '\n' + 'os.path.isjunction = lambda p: os.path.normcase(os.path.abspath(p)) == _t\n') + via_reparse_tag = ( + 'import os\n_t = ' + repr(target) + '\n_real = os.lstat\n' + 'class _S:\n' + ' def __init__(self, r):\n' + ' self._r = r\n' + ' self.st_reparse_tag = 0xA0000003\n' + ' def __getattr__(self, n):\n' + ' return getattr(self._r, n)\n' + 'def _lstat(p, *a, **k):\n' + ' r = _real(p, *a, **k)\n' + ' if isinstance(p, str) and os.path.normcase(os.path.abspath(p)) == _t:\n' + ' return _S(r)\n' + ' return r\n' + 'os.lstat = _lstat\n' + "if hasattr(os.path, 'isjunction'):\n" + ' del os.path.isjunction\n') + for preamble in (via_isjunction, via_reparse_tag): + result = setup_with_preamble(self.checkout, preamble, 'build_py') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a junction must be refused like a symlink') + self.assertIn('REFUSED: stale build output', output) + self.assertIn('build/lib/' + PACKAGE + ' (junction)', output) + self.assertEqual(sorted(junction.iterdir()), [], 'nothing may be written through the junction') + # A junction among the package sources is refused before any copy. + shutil.rmtree(self.checkout / 'build') + source = os.path.normcase(os.path.abspath(self.checkout / SOURCE_DIR)) + result = setup_with_preamble( + self.checkout, via_isjunction.replace(repr(target), repr(source)), 'build_py') + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0) + self.assertIn('REFUSED: symlinked source', output) + self.assertIn('src/' + PACKAGE + ' (junction)', output) + self.assertFalse((self.checkout / 'build' / 'lib' / PACKAGE).exists()) + # Without any simulated junction the same direct build is accepted. + result = setup_with_preamble(self.checkout, 'import os', 'build_py') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_pre_generated_dist_info_is_validated(self): + # ``bdist_wheel --dist-info-dir`` copies the given directory into the + # wheel as is (pip's build hook passes the one it generated), so it must + # hold only what this build's own metadata conversion would produce. + meta = self.base / 'meta' + meta.mkdir() + generated = direct_setup(self.checkout, 'dist_info', '--output-dir', str(meta)) + self.assertEqual(generated.returncode, 0, generated.stdout + generated.stderr) + dist_infos = sorted(meta.glob('*.dist-info')) + self.assertEqual(len(dist_infos), 1, dist_infos) + dist_info = dist_infos[0] + self.assertTrue((dist_info / 'METADATA').is_file()) + planted = dist_info / 'private.txt' + planted.write_text('private\n', encoding='utf-8') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist_wheel', '--dist-info-dir', str(dist_info), '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a pre-generated dist-info with an extra file must be refused') + self.assertIn('REFUSED: stale build output', output) + self.assertIn(os.path.relpath(planted, self.checkout).replace(os.sep, '/'), output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + self.assertTrue(planted.is_file(), 'the leftover is reported, not deleted') + # The directory as generated is accepted, and the wheel carries exactly it. + planted.unlink() + result = direct_setup(self.checkout, 'bdist_wheel', '--dist-info-dir', str(dist_info), '--dist-dir', str(dist)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + wheels = sorted(dist.glob('*.whl')) + self.assertEqual(len(wheels), 1, wheels) + self.assertEqual(wheel_package_files(wheels[0]), tracked_source_files()) + with zipfile.ZipFile(wheels[0]) as archive: + shipped = sorted(name.split('/', 1)[1] for name in archive.namelist() + if name.startswith(dist_info.name + '/')) + self.assertNotIn('private.txt', shipped) + self.assertIn('METADATA', shipped) + def test_hard_linked_build_outputs_are_refused(self): # A hard link at a produced path shares its bytes with another file. # setuptools' copy unlinks the destination before writing, so the other @@ -689,6 +816,14 @@ def test_install_lib_skip_build_validates_the_directory_it_copies_from(self): self.assertIn('elsewhere/' + PACKAGE + '/__init__.py (missing)', output) self.assertIn('under elsewhere', output) self.assertEqual(installed_files(self.target), [], 'nothing may be installed after a refusal') + # Without --skip-build the build rewrites build/lib, but the copy still + # comes from the named directory, which must be validated just the same. + result = direct_setup(self.checkout, 'install_lib', '--build-dir', 'elsewhere', + '--install-dir', str(self.target)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'the copied-from directory must be validated after a build too') + self.assertIn('elsewhere/' + PACKAGE + '/stale.py', output) + self.assertEqual(installed_files(self.target), []) result = direct_setup(self.checkout, 'install_lib', '--skip-build', '--build-dir', 'build/lib', '--install-dir', str(self.target)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) From e8406c30bae156a66f585642661d8f1ee44c3754 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 15:43:12 +0000 Subject: [PATCH 10/14] Compare a pre-generated dist-info with this build's own metadata, byte for byte bdist_wheel --dist-info-dir copies the given directory into the wheel as is. The previous check accepted any file whose name the metadata conversion writes, so an edited METADATA (for example an added Requires-Dist) shipped, and a missing top_level.txt went unnoticed. The directory is now compared with a dist-info that a separate `setup.py dist_info` run regenerates into a temporary directory: every name (exactly, so a case variant is another entry), every file's bytes, and every expected file must match; aliases, non-regular files and hard links are refused as before. WHEEL and RECORD are no longer accepted, since neither pip's metadata hook nor dist_info writes them. pip's own PEP 517 flow passes, with and without build isolation. Scientific effect: NONE. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 10 +-- setup.py | 120 ++++++++++++++++++++----------- tests/test_install_provenance.py | 46 +++++++++--- 3 files changed, 121 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index d4ef8eb..83fc371 100644 --- a/README.md +++ b/README.md @@ -63,10 +63,12 @@ run that does not recompile it, such as `--skip-build` or a build under `python -B`, is refused. Every `bdist` command (`bdist_wheel`, `bdist_egg`, `bdist_dumb`, `bdist_rpm`) refuses a leftover anywhere under the shared `build/bdist.*/` base (an interrupted build leaves one), the wheel command -also refuses a pre-generated `--dist-info-dir` holding anything its own -metadata conversion would not produce, and the wheel command repeats the -`build/lib/` check, so `--skip-build` cannot bypass it; with `--skip-build` -every expected output must be present with the bytes of its source. The +also refuses a pre-generated `--dist-info-dir` that differs in any name or +byte from this build's own metadata, which a separate `setup.py dist_info` run +regenerates for the comparison (`REFUSED: pre-generated metadata …`), and the +wheel command repeats the `build/lib/` check, so `--skip-build` cannot bypass +it; with `--skip-build` every expected output must be present with the bytes +of its source. The legacy `install`, `install_lib` and `bdist_egg` commands run the same check on the directory `install_lib` copies from (`--build-dir`), with or without `--skip-build`. The egg-info manifest is regenerated from the tree and a diff --git a/setup.py b/setup.py index 3edd8a5..cb321a0 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,9 @@ (``build/bdist.``, where the wheel, egg, dumb and rpm staging trees live; an interrupted build leaves one, and the next build archives whatever it holds) and under its own staging directory, refuses a - pre-generated ``--dist-info-dir`` holding anything its own metadata - conversion would not produce (the directory is copied into the wheel as + pre-generated ``--dist-info-dir`` that differs in any name or byte from this + build's own metadata, which a separate ``dist_info`` run of this script + regenerates for the comparison (the directory is copied into the wheel as is), and repeats the ``build/lib`` comparison, so ``--skip-build`` cannot bypass it. With ``--skip-build`` nothing is rebuilt, so the byte comparison runs there instead, every expected output must be present as a regular file, @@ -74,7 +75,9 @@ import filecmp import os import stat +import subprocess import sys +import tempfile from importlib import metadata as _metadata from setuptools import setup @@ -98,6 +101,7 @@ # defines the name on Windows builds; the value is fixed by the platform. _IO_REPARSE_TAG_MOUNT_POINT = getattr(stat, 'IO_REPARSE_TAG_MOUNT_POINT', 0xA0000003) REMEDY = 'git clean -fdX -- build ' + EGG_INFO +SETUP_SCRIPT = os.path.abspath(__file__) def _canonical(path: str) -> str: @@ -309,53 +313,85 @@ def _guard_skipped_build(command, tree: str | None = None) -> None: _guard_build_lib(command.get_finalized_command('build_py'), compare_bytes=True, recompiles=False, tree=tree) -def _dist_info_leftovers(tree: str, distribution) -> list[str]: - """Entries of a pre-generated dist-info directory that this build's own - metadata conversion would not produce, plus every alias or hard link. +def _interpreter_flags() -> list[str]: + """The running interpreter's isolation options, for a child run of this script.""" + options = (('isolated', '-I'), ('ignore_environment', '-E'), ('no_user_site', '-s'), + ('no_site', '-S'), ('safe_path', '-P'), ('dont_write_bytecode', '-B')) + return [option for flag, option in options if getattr(sys.flags, flag, False)] - ``bdist_wheel --dist-info-dir`` copies the directory into the wheel as is. - The conversion keeps the egg-info writers' files except ``PKG-INFO`` (which - becomes ``METADATA``), ``requires.txt`` and ``SOURCES.txt``, and places the - distribution's license files under ``licenses/``. - """ - written = {entry.name for entry in _metadata.entry_points(group='egg_info.writers')} - written -= {'PKG-INFO', 'requires.txt', 'SOURCES.txt'} - written |= {'METADATA', 'WHEEL', 'RECORD'} - licenses = {os.path.normpath(path).replace(os.sep, '/') - for path in (distribution.metadata.license_files or ())} + +def _fresh_dist_info(workdir: str) -> str: + """This build's own dist-info directory, regenerated by a separate + ``dist_info`` run of this script into the empty ``workdir``.""" + command = [sys.executable, *_interpreter_flags(), SETUP_SCRIPT, '-q', 'dist_info', '--output-dir', workdir] + try: + result = subprocess.run(command, cwd=os.path.dirname(SETUP_SCRIPT), capture_output=True, text=True, + errors='replace') + except OSError as error: + raise SetupError('REFUSED: could not regenerate this build\'s metadata to compare with ' + '--dist-info-dir: ' + str(error)) from error + produced = [name for name in os.listdir(workdir) if name.endswith('.dist-info')] + if result.returncode != 0 or len(produced) != 1: + raise SetupError('REFUSED: could not regenerate this build\'s metadata to compare with ' + '--dist-info-dir: ' + (result.stderr.strip()[-2000:] or 'no dist-info produced')) + return os.path.join(workdir, produced[0]) + + +def _entries(tree: str) -> tuple[set[str], set[str]]: + """Relative paths of the directories and of the files under ``tree``, as spelled on disk.""" + dirs, files = set(), set() + for dirpath, dirnames, filenames in os.walk(tree): + inside = os.path.relpath(dirpath, tree) + dirs.update(os.path.normpath(os.path.join(inside, name)) for name in dirnames) + files.update(os.path.normpath(os.path.join(inside, name)) for name in filenames) + return dirs, files + + +def _dist_info_mismatches(tree: str, reference: str) -> list[str]: + """Every way a pre-generated dist-info directory differs from ``reference``, + this build's own: an alias or hard link, a non-regular file, an entry the + reference lacks, bytes that differ, or a reference file that is missing. + Names are compared exactly, so a case variant counts as another entry.""" found = _symlinked_parents(tree) kind = _alias(tree) if kind: return sorted(found + [_relative(tree) + ' (' + kind + ')']) if not os.path.isdir(tree): return sorted(found + [_relative(tree) + ' (missing)']) - for dirpath, dirnames, filenames in os.walk(tree): - inside = os.path.relpath(dirpath, tree).replace(os.sep, '/') - for name in dirnames: - path = os.path.join(dirpath, name) - kind = _alias(path) - if kind: - found.append(_relative(path) + ' (' + kind + ')') - elif not (inside == os.curdir and name == 'licenses'): - found.append(_relative(path) + ' (directory)') - for name in filenames: - path = os.path.join(dirpath, name) - kind = _alias(path) - if kind: - found.append(_relative(path) + ' (' + kind + ')') - elif os.stat(path).st_nlink > 1: - found.append(_relative(path) + ' (hard link)') - elif inside == os.curdir: - if name not in written: - found.append(_relative(path)) - elif inside == 'licenses': - if name not in licenses: - found.append(_relative(path)) - else: - found.append(_relative(path)) + expected_dirs, expected_files = _entries(reference) + dirs, files = _entries(tree) + for entry in sorted(dirs): + path = os.path.join(tree, entry) + kind = _alias(path) + if kind: + found.append(_relative(path) + ' (' + kind + ')') + elif entry not in expected_dirs: + found.append(_relative(path) + ' (not in this build\'s metadata)') + for entry in sorted(files): + path = os.path.join(tree, entry) + kind = _alias(path) + if kind: + found.append(_relative(path) + ' (' + kind + ')') + elif not os.path.isfile(path): + found.append(_relative(path) + ' (not a regular file)') + elif os.stat(path).st_nlink > 1: + found.append(_relative(path) + ' (hard link)') + elif entry not in expected_files: + found.append(_relative(path) + ' (not in this build\'s metadata)') + elif not filecmp.cmp(os.path.join(reference, entry), path, shallow=False): + found.append(_relative(path) + ' (bytes differ from this build\'s metadata)') + for entry in sorted(expected_files - files): + found.append(_relative(os.path.join(tree, entry)) + ' (missing)') return sorted(found) +def _refuse_dist_info(mismatches: list[str]) -> None: + if mismatches: + raise SetupError( + 'REFUSED: pre-generated metadata does not match this build\'s own: ' + ', '.join(mismatches) + + '. Regenerate it from this checkout (setup.py dist_info) or build without --dist-info-dir.') + + def _guard_staging(command, staging_dir: str) -> None: """Refuse any leftover under the shared bdist base (``build/bdist.``), where the wheel, egg, dumb and rpm staging trees live, and under the @@ -386,8 +422,10 @@ def run(self) -> None: _guard_staging(self, self.bdist_dir) if self.dist_info_dir: # Pre-generated metadata is copied into the wheel as is, so it must - # hold only what this build's own conversion would produce. - _refuse(_dist_info_leftovers(self.dist_info_dir, self.distribution), self.dist_info_dir) + # match, in names and bytes, what this build's own conversion + # produces; a separate run regenerates that for comparison. + with tempfile.TemporaryDirectory() as workdir: + _refuse_dist_info(_dist_info_mismatches(self.dist_info_dir, _fresh_dist_info(workdir))) if self.skip_build: _guard_skipped_build(self) else: diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index 8be6ad7..e2dc745 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -392,7 +392,8 @@ def test_windows_junctions_are_treated_as_symlinks(self): def test_pre_generated_dist_info_is_validated(self): # ``bdist_wheel --dist-info-dir`` copies the given directory into the # wheel as is (pip's build hook passes the one it generated), so it must - # hold only what this build's own metadata conversion would produce. + # match this build's own metadata, regenerated separately, in names and + # bytes. meta = self.base / 'meta' meta.mkdir() generated = direct_setup(self.checkout, 'dist_info', '--output-dir', str(meta)) @@ -400,19 +401,42 @@ def test_pre_generated_dist_info_is_validated(self): dist_infos = sorted(meta.glob('*.dist-info')) self.assertEqual(len(dist_infos), 1, dist_infos) dist_info = dist_infos[0] - self.assertTrue((dist_info / 'METADATA').is_file()) + metadata = dist_info / 'METADATA' + top_level = dist_info / 'top_level.txt' + original = {path: path.read_bytes() for path in (metadata, top_level)} + dist = self.base / 'dist' + + def refused(change: str, path: Path, label: str) -> None: + result = direct_setup(self.checkout, 'bdist_wheel', '--dist-info-dir', str(dist_info), '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a pre-generated dist-info with ' + change + ' must be refused') + self.assertIn('REFUSED: pre-generated metadata', output) + self.assertIn(os.path.relpath(path, self.checkout).replace(os.sep, '/') + label, output) + self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + planted = dist_info / 'private.txt' planted.write_text('private\n', encoding='utf-8') - dist = self.base / 'dist' - result = direct_setup(self.checkout, 'bdist_wheel', '--dist-info-dir', str(dist_info), '--dist-dir', str(dist)) - output = result.stdout + result.stderr - self.assertNotEqual(result.returncode, 0, 'a pre-generated dist-info with an extra file must be refused') - self.assertIn('REFUSED: stale build output', output) - self.assertIn(os.path.relpath(planted, self.checkout).replace(os.sep, '/'), output) - self.assertEqual(sorted(dist.glob('*.whl')) if dist.exists() else [], []) + refused('an extra file', planted, " (not in this build's metadata)") self.assertTrue(planted.is_file(), 'the leftover is reported, not deleted') - # The directory as generated is accepted, and the wheel carries exactly it. planted.unlink() + # Edited bytes at a name the conversion writes are not this build's metadata. + metadata.write_bytes(original[metadata] + b'Requires-Dist: injected-dependency\n') + refused('edited METADATA', metadata, " (bytes differ from this build's metadata)") + metadata.write_bytes(original[metadata]) + top_level.unlink() + refused('a missing file', top_level, ' (missing)') + # A case variant is another name, even where the filesystem would not tell. + variant = dist_info / 'Top_Level.txt' + variant.write_bytes(original[top_level]) + refused('a case variant of a file', variant, " (not in this build's metadata)") + variant.unlink() + top_level.write_bytes(original[top_level]) + # bdist_wheel writes WHEEL and RECORD itself; the conversion produces neither. + wheel_file = dist_info / 'WHEEL' + wheel_file.write_text('Wheel-Version: 1.0\n', encoding='utf-8') + refused('a WHEEL file', wheel_file, " (not in this build's metadata)") + wheel_file.unlink() + # The directory as generated is accepted, and the wheel carries exactly it. result = direct_setup(self.checkout, 'bdist_wheel', '--dist-info-dir', str(dist_info), '--dist-dir', str(dist)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) wheels = sorted(dist.glob('*.whl')) @@ -421,8 +445,10 @@ def test_pre_generated_dist_info_is_validated(self): with zipfile.ZipFile(wheels[0]) as archive: shipped = sorted(name.split('/', 1)[1] for name in archive.namelist() if name.startswith(dist_info.name + '/')) + shipped_metadata = archive.read(dist_info.name + '/METADATA') self.assertNotIn('private.txt', shipped) self.assertIn('METADATA', shipped) + self.assertEqual(shipped_metadata, original[metadata]) def test_hard_linked_build_outputs_are_refused(self): # A hard link at a produced path shares its bytes with another file. From 9b3d500bd42e33c0f64a2d74ab48511451ab91ae Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 16:12:54 +0000 Subject: [PATCH 11/14] Close bypasses and false refusals found in pre-push reads of 7191e599 Scientific effect: NONE. The fixes: - A symlink among the files the manifest names is refused when egg_info builds the manifest. The metadata conversion reads README.md into METADATA and copies LICENSE into the wheel, following the link; sdist already refused it, wheels did not. - A path spelled with a trailing separator or "." is checked as the link it names, and a symlinked directory before "..". Before this, `--build-lib bl/` with a symlinked `bl` wrote through it. - install_lib admits bytecode only in build/lib, right after the same run recompiled it. A cache file in another --build-dir was compiled by nobody, and the copy keeps a destination that is newer than its source. - editable_wheel compares a pre-generated --dist-info-dir with this build's own metadata, as bdist_wheel does. pip's editable install passes one, and the command copies it into the wheel as is. build_py no longer guards in editable mode: nothing is copied there, and the false refusal surfaced as a swallowed warning. - Each bdist command checks only its own staging tree, the one it archives. The shared-base rule refused every later pip wheel after a successful bdist_rpm, which leaves its rpm tree behind. - A tree kept earlier in the same run is admitted, which fixes `bdist --formats=zip,gztar`. - bdist_rpm now subclasses setuptools' command rather than distutils'. It forces its copies and admits the spec and source archive that every run rewrites. - Walks do not descend into a Windows junction they report. - The dry-run control is skipped on setuptools 81 and later, which removed the global --dry-run. The output directory (dist/ or --dist-dir) is taken as given. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 62 +++++---- setup.py | 174 ++++++++++++++++++------- tests/test_install_provenance.py | 213 +++++++++++++++++++++++++++++-- 3 files changed, 366 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 83fc371..8fcd712 100644 --- a/README.md +++ b/README.md @@ -61,31 +61,43 @@ Bytecode under `build/lib/` is accepted only from the run that compiles it (`build_py --compile` with bytecode writing enabled); a cache file found by a run that does not recompile it, such as `--skip-build` or a build under `python -B`, is refused. Every `bdist` command (`bdist_wheel`, `bdist_egg`, -`bdist_dumb`, `bdist_rpm`) refuses a leftover anywhere under the shared -`build/bdist.*/` base (an interrupted build leaves one), the wheel command -also refuses a pre-generated `--dist-info-dir` that differs in any name or -byte from this build's own metadata, which a separate `setup.py dist_info` run -regenerates for the comparison (`REFUSED: pre-generated metadata …`), and the -wheel command repeats the `build/lib/` check, so `--skip-build` cannot bypass -it; with `--skip-build` every expected output must be present with the bytes -of its source. The -legacy `install`, `install_lib` and `bdist_egg` commands run the same check -on the directory `install_lib` copies from (`--build-dir`), with or without -`--skip-build`. The egg-info manifest is regenerated from the tree and a -stale `SOURCES.txt` is never read back, so it can add a file neither to a wheel -nor to a `setup.py sdist` archive; any other file left in the egg-info -directory is refused, because an sdist and a direct `bdist_wheel` would copy -it into the archive, and so is a leftover sdist release tree -(`universal_law_query-0.1.0/`), which an sdist would archive whole, and a -symlink among the files an sdist's manifest names. A symlink or hard link -in or at the egg-info directory, or a symlinked `build/`, is refused before -anything is written, because setuptools' metadata writers and build copies -would otherwise write through to whatever the link points at. A hard link -under `build/lib/` is refused because its bytes are shared with another path, -so the tree is not this build's output; the build copy itself unlinks its -destination before writing and does not write through. A Windows junction -counts as a symlink throughout. A `--dry-run` build writes nothing and is not -refused for that; leftovers are still refused. +`bdist_dumb`, `bdist_rpm`) refuses a leftover in its own staging tree under +`build/bdist.*/`, which it archives as found (an interrupted build leaves +one). It does not refuse a tree kept by an earlier command of the same run, as +`bdist` does between two formats, because that tree is the run's own output. +For `bdist_rpm`, the spec and source archive that every run rewrites, and a +successful run leaves behind, are not leftovers. A sibling command's tree is +never read. The wheel and editable-wheel commands also refuse a +pre-generated `--dist-info-dir` that differs in any name or byte from this +build's own metadata, which a separate `setup.py dist_info` run regenerates +for the comparison (`REFUSED: pre-generated metadata …`), and the wheel +command repeats the `build/lib/` check, so `--skip-build` cannot bypass it; +with `--skip-build` every expected output must be present with the bytes of +its source. The legacy `install`, `install_lib` and `bdist_egg` commands run +the same check on the directory `install_lib` copies from (`--build-dir`), +with or without `--skip-build`; bytecode there is accepted only in +`build/lib/` right after the same run recompiled it. The egg-info manifest is +regenerated from the tree and a stale `SOURCES.txt` is never read back, so it +can add a file neither to a wheel nor to a `setup.py sdist` archive; any other +file left in the egg-info directory is refused, because an sdist and a direct +`bdist_wheel` would copy it into the archive, and so is a leftover sdist +release tree (`universal_law_query-0.1.0/`), which an sdist would archive +whole. A symlink among the files the manifest names (for example `LICENSE` or +`README.md`, which the wheel's metadata copies or embeds) is refused whenever +the metadata is generated, for a wheel, an sdist or an install alike. A +symlink or hard link in or at the egg-info directory, or a symlinked `build/`, +is refused before anything is written, +because setuptools' metadata writers and build copies would otherwise write +through to whatever the link points at; a directory named with a trailing `/` +or `/.`, or reached through `..` after a symlinked directory, counts as the +link. A hard link under `build/lib/` is refused because its bytes are shared +with another path, so the tree is not this build's output; the build copy +itself unlinks its destination before writing and does not write through. A +Windows junction counts as a symlink throughout. A `--dry-run` build (an option +setuptools 81 removed) writes nothing and is not refused for that; leftovers +are still refused. An editable +install copies nothing and runs from `src/`. The output directory (`dist/` or +`--dist-dir`) is taken as given. `pyproject.toml` also declares that the package ships no package data. Nothing is deleted for you; remove the leftovers before installing: `git clean -fdX -- build src/universal_law_query.egg-info`. diff --git a/setup.py b/setup.py index cb321a0..c17877b 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools reuses ``build/`` between builds and removes nothing from it. A module left under ``build/lib`` by an install from another commit is copied into the next wheel although ``src/`` does not contain it, and an import check -of the installed package still passes. Eight commands are guarded: +of the installed package still passes. Nine commands are guarded: - ``build_py`` refuses a symlink among the package sources it would copy, or a symlinked directory above one, because the copy and the byte comparison @@ -15,7 +15,8 @@ before copying anything and again afterwards, and refuses when anything else is present. Every symlink under the tree is refused wherever it points, because the walk does not follow symlinks but the wheel staging step - dereferences them (a Windows junction counts as a symlink throughout); a + dereferences them (a Windows junction counts as a symlink throughout, and no + walk descends into one); a hard-linked file is refused as well, since its bytes are shared with another path. The run rewrites every file it produces (setuptools would otherwise skip a destination that is newer than its source), and afterwards @@ -25,11 +26,11 @@ writing enabled); under ``python -B`` or ``PYTHONDONTWRITEBYTECODE`` setuptools skips the compile step, so a cache file found under ``build/lib`` is refused. A dry run writes nothing, so the comparison after the run is skipped there; - the leftover check before it still applies. -- ``bdist_wheel`` refuses a leftover anywhere under the shared bdist base - (``build/bdist.``, where the wheel, egg, dumb and rpm staging - trees live; an interrupted build leaves one, and the next build archives - whatever it holds) and under its own staging directory, refuses a + the leftover check before it still applies. An editable build copies + nothing (the install runs from ``src/``), so there is nothing to guard. +- ``bdist_wheel`` refuses a leftover anywhere in its own staging tree (by + default ``build/bdist./wheel``; an interrupted build leaves one, + and the next build archives whatever it holds), refuses a pre-generated ``--dist-info-dir`` that differs in any name or byte from this build's own metadata, which a separate ``dist_info`` run of this script regenerates for the comparison (the directory is copied into the wheel as @@ -37,13 +38,23 @@ bypass it. With ``--skip-build`` nothing is rebuilt, so the byte comparison runs there instead, every expected output must be present as a regular file, and bytecode, which nothing recompiles, is refused. +- ``editable_wheel`` applies the same ``--dist-info-dir`` comparison; pip's + editable install passes the metadata it generated, and the command copies it + into the editable wheel as is. - ``install_lib`` applies the same comparison to the directory it copies from (its ``--build-dir``, by default ``build/lib``), whether or not it rebuilt it, which covers the legacy ``install``, ``install_lib`` and ``bdist_egg`` - commands with and without ``--skip-build``. -- ``bdist_egg``, ``bdist_dumb`` and ``bdist_rpm`` refuse a leftover under the - shared bdist base and their own staging trees the same way; the legacy - archives are assembled from whatever those trees hold. + commands with and without ``--skip-build``. Bytecode is admitted only in + ``build/lib`` after this run recompiled it there; in another directory + nothing recompiled it, and the copy keeps a cache file newer than its source. +- ``bdist_egg`` and ``bdist_dumb`` refuse a leftover in their own staging + trees the same way; the legacy archives are assembled from whatever those + trees hold. ``bdist_rpm`` refuses one in its rpm tree, except the spec and + source archive that every run rewrites (forced) and a successful run leaves + behind with empty directories. Each command reads only its own tree, so a + sibling's tree under ``build/bdist.`` blocks nothing else. A tree + that an earlier command of the same run kept (``bdist`` keeps one between + two formats of a command) is that run's own output and is admitted. - ``egg_info`` regenerates ``SOURCES.txt`` from the tree and never reads a leftover manifest back in, so a stale entry can add a file neither to a wheel nor to a source distribution; any other file left beside the ones this run @@ -53,7 +64,10 @@ would follow a symlink, overwriting whatever it points at, so the directory itself and every entry in it must be a regular file with a single link; a hard-linked file shares its bytes with another path and would be written - through just the same. + through just the same. A symlink among the files the manifest names, or a + symlinked directory above one, is refused when the manifest is built: the + metadata conversion reads ``README.md`` into ``METADATA`` and copies the + license files into the wheel, following any link. - ``sdist`` refuses a symlink among the files the manifest names, or a symlinked directory above one, before copying anything: the release tree would otherwise hold the link's target as a regular file. It also refuses a @@ -64,7 +78,9 @@ A symlinked directory above any of these trees (``build/`` above ``build/lib`` and the staging tree, ``src/`` above the egg-info directory) is refused before anything is written, because every write under it would land wherever the link -points. +points. A tree named with a trailing separator or ``.`` is checked as the link +it names, and one reached through ``..`` after a symlinked directory is refused +as well. The output directory (``dist/`` or ``--dist-dir``) is taken as given. Nothing is deleted; each refusal names the leftovers and the remedy, which is ``git clean`` for the checkout's own ``build/`` and egg-info directory and @@ -81,15 +97,16 @@ from importlib import metadata as _metadata from setuptools import setup -# The two legacy commands live in distutils; import them through the name -# setuptools itself uses. Importing ``setuptools._distutils.…`` directly would -# load second copies of those modules and rebind the shared package's -# attributes to unpatched classes. +# setuptools has no bdist_dumb of its own and runs distutils' command; import +# it through the name setuptools itself uses. Importing +# ``setuptools._distutils.…`` directly would load a second copy of that module +# and rebind the shared package's attributes to unpatched classes. from distutils.command.bdist_dumb import bdist_dumb as _bdist_dumb # noqa: I001 -from distutils.command.bdist_rpm import bdist_rpm as _bdist_rpm from setuptools.command.bdist_egg import bdist_egg as _bdist_egg +from setuptools.command.bdist_rpm import bdist_rpm as _bdist_rpm from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel from setuptools.command.build_py import build_py as _build_py +from setuptools.command.editable_wheel import editable_wheel as _editable_wheel from setuptools.command.egg_info import egg_info as _egg_info from setuptools.command.egg_info import manifest_maker as _manifest_maker from setuptools.command.install_lib import install_lib as _install_lib @@ -121,8 +138,10 @@ def _alias(path: str) -> str | None: A Windows junction is a directory reparse point that ``os.path.islink`` does not report; Python 3.12 has ``os.path.isjunction``, and older - interpreters expose the reparse tag on ``os.lstat``. + interpreters expose the reparse tag on ``os.lstat``. A trailing separator + or ``.`` would make the system follow the link, so it is dropped first. """ + path = os.path.normpath(path) if os.path.islink(path): return 'symlink' isjunction = getattr(os.path, 'isjunction', None) @@ -148,10 +167,11 @@ def _unexpected(tree: str, produced: set[str], empty_dirs: bool = False) -> list for dirpath, dirnames, filenames in os.walk(tree): if empty_dirs and dirpath != tree and not dirnames and not filenames: found.append(_relative(dirpath) + ' (empty directory)') - for name in dirnames: + for name in list(dirnames): kind = _alias(os.path.join(dirpath, name)) if kind: found.append(_relative(os.path.join(dirpath, name)) + ' (' + kind + ')') + dirnames.remove(name) # os.walk would descend into a Windows junction for name in filenames: path = os.path.join(dirpath, name) kind = _alias(path) @@ -178,7 +198,7 @@ def _refuse(unexpected: list[str], tree: str) -> None: if unexpected: raise SetupError( 'REFUSED: stale build output not produced from this source tree: ' - + ', '.join(unexpected) + + ', '.join(sorted(set(unexpected))) + '. Remove build leftovers before installing: ' + _remedy(tree)) @@ -221,12 +241,14 @@ def _symlinked_sources(build_py_command) -> list[str]: def _symlinked_parents(path: str) -> list[str]: """Directories above a relative ``path`` that are symlinks; a write under - ``path`` would land wherever they point. An absolute path, configured - deliberately, is taken as given.""" + ``path`` would land wherever they point. Each directory is taken as + written, since ``..`` after a symlinked one leaves the link's target and + normalizing would hide it. An absolute path, configured deliberately, is + taken as given.""" found = [] if os.path.isabs(path): return found - parent = os.path.dirname(os.path.normpath(path)) + parent = os.path.dirname(path) while parent and parent != os.path.dirname(parent): kind = _alias(parent) if kind: @@ -344,6 +366,8 @@ def _entries(tree: str) -> tuple[set[str], set[str]]: inside = os.path.relpath(dirpath, tree) dirs.update(os.path.normpath(os.path.join(inside, name)) for name in dirnames) files.update(os.path.normpath(os.path.join(inside, name)) for name in filenames) + # Reported as entries; os.walk would descend into a Windows junction. + dirnames[:] = [name for name in dirnames if not _alias(os.path.join(dirpath, name))] return dirs, files @@ -392,19 +416,47 @@ def _refuse_dist_info(mismatches: list[str]) -> None: + '. Regenerate it from this checkout (setup.py dist_info) or build without --dist-info-dir.') -def _guard_staging(command, staging_dir: str) -> None: - """Refuse any leftover under the shared bdist base (``build/bdist.``), - where the wheel, egg, dumb and rpm staging trees live, and under the - command's own staging directory when it was configured elsewhere.""" - base = command.get_finalized_command('bdist').bdist_base - for tree in (base, staging_dir): - _refuse(sorted(_symlinked_parents(tree) + _unexpected(tree, set(), empty_dirs=True)), tree) +def _guard_dist_info(dist_info_dir: str | None) -> None: + """A given ``--dist-info-dir`` is copied into the wheel as is, so it must + match, in names and bytes, what this build's own conversion produces; a + separate run regenerates that for the comparison.""" + if dist_info_dir: + with tempfile.TemporaryDirectory() as workdir: + _refuse_dist_info(_dist_info_mismatches(os.fspath(dist_info_dir), _fresh_dist_info(workdir))) + + +# Files of staging trees that an earlier command of this same run kept +# (``bdist`` keeps the tree between two formats of one command): this run's +# own output, not leftovers. +_KEPT_STAGING: set[str] = set() + + +def _guard_staging(staging_dir: str, rewritten: frozenset[str] = frozenset(), empty_dirs: bool = True) -> None: + """Refuse any leftover in a command's own staging tree, which the command + archives (or, for rpm, hands to rpmbuild) as found. A sibling command's + tree under the same ``build/bdist.`` is never read. Files that an + earlier command of this same run kept, and ``rewritten`` files that the + command overwrites on every run, are admitted.""" + _refuse(sorted(_symlinked_parents(staging_dir) + + _unexpected(staging_dir, _KEPT_STAGING | rewritten, empty_dirs=empty_dirs)), staging_dir) + + +def _remember_kept(command, staging_dir: str) -> None: + """After a command that keeps its staging tree, admit that tree's files to + later commands of this run.""" + if getattr(command, 'keep_temp', False) and os.path.isdir(staging_dir): + for dirpath, _dirnames, filenames in os.walk(staging_dir): + _KEPT_STAGING.update(_canonical(os.path.join(dirpath, name)) for name in filenames) class build_py(_build_py): """Refuse build output that this source tree did not produce.""" def run(self) -> None: + if self.editable_mode: + # An editable build copies nothing (the install runs from src/), + # so there is no build tree to guard. + return super().run() _guard_build_lib(self) # Rewrite every output: a leftover at a produced path that is newer than # its source would otherwise be kept as "up to date" and shipped. @@ -419,53 +471,73 @@ class bdist_wheel(_bdist_wheel): """Refuse a leftover staging tree, and re-check build/lib even with --skip-build.""" def run(self) -> None: - _guard_staging(self, self.bdist_dir) - if self.dist_info_dir: - # Pre-generated metadata is copied into the wheel as is, so it must - # match, in names and bytes, what this build's own conversion - # produces; a separate run regenerates that for comparison. - with tempfile.TemporaryDirectory() as workdir: - _refuse_dist_info(_dist_info_mismatches(self.dist_info_dir, _fresh_dist_info(workdir))) + _guard_staging(self.bdist_dir) + _guard_dist_info(self.dist_info_dir) if self.skip_build: _guard_skipped_build(self) else: # The build rewrites build/lib next and checks it again itself. _guard_build_lib(self.get_finalized_command('build_py')) super().run() + _remember_kept(self, self.bdist_dir) + + +class editable_wheel(_editable_wheel): + """Validate a pre-generated dist-info; pip's editable flow passes the one it generated.""" + + def run(self) -> None: + # Before the run: it writes WHEEL into the given directory, then copies + # the directory into the wheel as is. + _guard_dist_info(self.dist_info_dir) + super().run() class bdist_egg(_bdist_egg): """Refuse a leftover egg staging tree; the legacy archive is assembled from whatever it holds.""" def run(self) -> None: - _guard_staging(self, self.bdist_dir) + _guard_staging(self.bdist_dir) super().run() + _remember_kept(self, self.bdist_dir) class bdist_dumb(_bdist_dumb): """Refuse a leftover dumb staging tree; the archive is the pseudo-installation tree as found.""" def run(self) -> None: - _guard_staging(self, self.bdist_dir) + _guard_staging(self.bdist_dir) super().run() + _remember_kept(self, self.bdist_dir) class bdist_rpm(_bdist_rpm): - """Refuse a leftover rpm build tree under the bdist base before rpmbuild sees it.""" + """Refuse a leftover in the rpm build tree before rpmbuild sees it.""" def run(self) -> None: - _guard_staging(self, self.rpm_base) + # Every run rewrites the spec and the source archive that rpmbuild + # reads, and a successful run leaves both behind with empty + # directories; the copy is forced, so an existing archive newer than + # the new one is not kept. Anything else in the tree is a leftover. + self.force = True + archive = self.distribution.get_fullname() + ('.tar.bz2' if self.use_bzip2 else '.tar.gz') + rewritten = frozenset({ + _canonical(os.path.join(self.rpm_base, 'SPECS', self.distribution.get_name() + '.spec')), + _canonical(os.path.join(self.rpm_base, 'SOURCES', archive))}) + _guard_staging(self.rpm_base, rewritten, empty_dirs=False) super().run() + _remember_kept(self, self.rpm_base) class install_lib(_install_lib): """Validate the directory this command copies from (--build-dir, by default build/lib).""" def install(self): - # Runs after build(): bytecode a build in this run wrote is admitted; - # under --skip-build nothing recompiles it. A dry run copies nothing. - _guard_build_lib(self.get_finalized_command('build_py'), compare_bytes=not self.dry_run, - recompiles=not self.skip_build, tree=self.build_dir) + # Runs after build(): bytecode a build in this run wrote is admitted, + # but only in build/lib, the tree that build recompiled; under + # --skip-build nothing recompiles it. A dry run copies nothing. + build_py = self.get_finalized_command('build_py') + recompiled = not self.skip_build and _canonical(self.build_dir) == _canonical(build_py.build_lib) + _guard_build_lib(build_py, compare_bytes=not self.dry_run, recompiles=recompiled, tree=self.build_dir) return super().install() @@ -486,6 +558,10 @@ def find_sources(self) -> None: mm.manifest = manifest_filename mm.run() self.filelist = mm.filelist + # The metadata conversion reads README.md into METADATA and copies the + # license files into the wheel, following any symlink, so every file + # the manifest names must be a file of this tree, as for an sdist. + _refuse_sources(_symlinked_paths(self.filelist.files)) def run(self) -> None: written = {entry.name for entry in _metadata.entry_points(group='egg_info.writers')} @@ -513,6 +589,6 @@ def make_release_tree(self, base_dir, files) -> None: super().make_release_tree(base_dir, files) -setup(cmdclass={'build_py': build_py, 'bdist_wheel': bdist_wheel, 'bdist_egg': bdist_egg, - 'bdist_dumb': bdist_dumb, 'bdist_rpm': bdist_rpm, 'install_lib': install_lib, - 'egg_info': egg_info, 'sdist': sdist}) +setup(cmdclass={'build_py': build_py, 'bdist_wheel': bdist_wheel, 'editable_wheel': editable_wheel, + 'bdist_egg': bdist_egg, 'bdist_dumb': bdist_dumb, 'bdist_rpm': bdist_rpm, + 'install_lib': install_lib, 'egg_info': egg_info, 'sdist': sdist}) diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index e2dc745..3add5c1 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -284,6 +284,17 @@ def test_stale_wheel_staging_tree_is_refused(self): self.assertIn(empty.relative_to(self.checkout).as_posix() + ' (empty directory)', output) self.assertIn(staged.parent.relative_to(self.checkout).as_posix() + ' (empty directory)', output) self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + # A staging directory configured elsewhere is the tree that is archived, + # so it is checked the same way and the refusal names that directory. + leftover = self.checkout / 'stage' / PACKAGE / 'stage_stale.py' + leftover.parent.mkdir(parents=True) + leftover.write_text('STAGE_STALE = True\n', encoding='utf-8') + result = direct_setup(self.checkout, 'bdist_wheel', '--bdist-dir', 'stage', '--dist-dir', str(self.wheels)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a leftover in a configured staging directory must be refused') + self.assertIn('stage/' + PACKAGE + '/stage_stale.py', output) + self.assertIn('under stage', output) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) def test_stale_bdist_egg_staging_tree_is_refused(self): # The legacy egg is assembled from build/bdist./egg, which a @@ -309,10 +320,10 @@ def test_stale_bdist_egg_staging_tree_is_refused(self): self.assertEqual(wheel_package_files(eggs[0]), tracked_source_files()) def test_stale_bdist_dumb_and_rpm_staging_trees_are_refused(self): - # Every bdist command stages under build/bdist.; the dumb - # archive is that pseudo-installation tree as found, and rpmbuild works - # from the rpm tree, so a leftover under the shared base is refused by - # each of them before anything is assembled. + # Each bdist command stages in its own tree under build/bdist.; + # the dumb archive is that pseudo-installation tree as found, and + # rpmbuild works from the rpm tree, so a leftover there is refused + # before anything is assembled. No command reads a sibling's tree. base = wheel_staging_dir(self.checkout).parent staged = base / 'dumb' / 'private' / 'stale.txt' staged.parent.mkdir(parents=True) @@ -325,21 +336,46 @@ def test_stale_bdist_dumb_and_rpm_staging_trees_are_refused(self): self.assertIn(staged.relative_to(self.checkout).as_posix(), output) self.assertEqual(sorted(dist.glob('*')) if dist.exists() else [], []) self.assertTrue(staged.is_file(), 'the leftover is reported, not deleted') - # The same leftover blocks the other bdist commands, which share the base. + # The wheel stages in its own tree and never archives the dumb one. + result = self.wheel() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + rpm_staged = base / 'rpm' / 'BUILD' / 'stale.txt' + rpm_staged.parent.mkdir(parents=True) + rpm_staged.write_text('STALE\n', encoding='utf-8') result = direct_setup(self.checkout, 'bdist_rpm', '--dist-dir', str(dist)) output = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0) - self.assertIn(staged.relative_to(self.checkout).as_posix(), output) + self.assertIn(rpm_staged.relative_to(self.checkout).as_posix(), output) self.assertNotIn('rpmbuild', output, 'the refusal must come before rpmbuild is involved') - result = self.wheel() - self.assertNotEqual(result.returncode, 0) - self.assertIn(staged.relative_to(self.checkout).as_posix(), result.stdout + result.stderr) # A clean dumb build is not refused. shutil.rmtree(base) result = direct_setup(self.checkout, 'bdist_dumb', '--format', 'zip', '--dist-dir', str(dist)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(len(sorted(dist.glob('*.zip'))), 1) + def test_what_a_successful_bdist_rpm_leaves_is_not_a_leftover(self): + # distutils' bdist_rpm keeps its rpm tree: the spec and the source + # archive it rewrites on every run, and empty directories. Neither a + # later wheel nor the next bdist_rpm may refuse those; anything else + # in the tree is still a leftover. + rpm = wheel_staging_dir(self.checkout).parent / 'rpm' + for name in ('BUILD', 'RPMS/noarch', 'SRPMS', 'SPECS', 'SOURCES'): + (rpm / name).mkdir(parents=True) + (rpm / 'SPECS' / 'universal-law-query.spec').write_text('%define name universal-law-query\n', encoding='utf-8') + (rpm / 'SOURCES' / 'universal_law_query-0.1.0.tar.gz').write_bytes(b'an earlier source archive') + result = self.wheel() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + spec_dir = self.base / 'spec' + result = direct_setup(self.checkout, 'bdist_rpm', '--spec-only', '--dist-dir', str(spec_dir)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(sorted(path.name for path in spec_dir.iterdir()), ['universal-law-query.spec']) + stale = rpm / 'SOURCES' / 'private.txt' + stale.write_text('PRIVATE\n', encoding='utf-8') + result = direct_setup(self.checkout, 'bdist_rpm', '--spec-only', '--dist-dir', str(spec_dir)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'another file in the rpm tree is a leftover') + self.assertIn(stale.relative_to(self.checkout).as_posix(), output) + def test_windows_junctions_are_treated_as_symlinks(self): # A Windows junction is not reported by os.path.islink; Python 3.12+ # reports it through os.path.isjunction and every supported version @@ -436,6 +472,29 @@ def refused(change: str, path: Path, label: str) -> None: wheel_file.write_text('Wheel-Version: 1.0\n', encoding='utf-8') refused('a WHEEL file', wheel_file, " (not in this build's metadata)") wheel_file.unlink() + # Under licenses/ only the declared license files belong, and no other + # directory belongs anywhere. + stray_license = dist_info / 'licenses' / 'private.txt' + stray_license.write_text('private\n', encoding='utf-8') + refused('an undeclared file under licenses/', stray_license, " (not in this build's metadata)") + stray_license.unlink() + extra_dir = dist_info / 'extra' + extra_dir.mkdir() + refused('an extra directory', extra_dir, " (not in this build's metadata)") + extra_dir.rmdir() + # A link is refused even to identical bytes: the copy would follow it, + # or the bytes are shared with another path. + outside = self.base / 'outside_top_level.txt' + outside.write_bytes(original[top_level]) + top_level.unlink() + for kind, make in (('symlink', os.symlink), ('hard link', os.link)): + try: + make(outside, top_level) + except (OSError, NotImplementedError): + continue # not available on this filesystem + refused('a ' + kind + 'ed entry', top_level, ' (' + kind + ')') + top_level.unlink() + top_level.write_bytes(original[top_level]) # The directory as generated is accepted, and the wheel carries exactly it. result = direct_setup(self.checkout, 'bdist_wheel', '--dist-info-dir', str(dist_info), '--dist-dir', str(dist)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -813,6 +872,13 @@ def test_hard_linked_egg_info_entries_are_refused_before_anything_is_written(sel def test_dry_run_build_is_not_refused_for_writing_nothing(self): # ``--dry-run`` writes nothing by design, so the post-run comparison # has nothing to compare and is skipped; the leftover check still runs. + probe = subprocess.run( + [sys.executable, '-c', 'from setuptools.dist import Distribution; ' + "print('dry-run' in [option[0] for option in Distribution.global_options])"], + capture_output=True, text=True, timeout=PIP_TIMEOUT) + self.assertEqual(probe.returncode, 0, probe.stderr) + if probe.stdout.strip() != 'True': + self.skipTest('this setuptools has no global --dry-run (removed in setuptools 81)') result = direct_setup(self.checkout, '--dry-run', 'build_py') output = result.stdout + result.stderr self.assertEqual(result.returncode, 0, output) @@ -924,6 +990,135 @@ def test_symlinked_build_directory_is_refused(self): self.assertIn('build (symlink)', output) self.assertEqual(sorted(outside.iterdir()), []) + def test_symlinked_license_or_readme_is_refused_for_wheels(self): + # The wheel's metadata embeds README.md as the long description and + # copies LICENSE under licenses/, so a symlink at either would ship its + # target; every file the manifest names is checked, as for an sdist. + private = self.base / 'private.txt' + private.write_text('PRIVATE\n', encoding='utf-8') + for name in ('LICENSE', 'README.md'): + with self.subTest(name=name): + path = self.checkout / name + original = path.read_bytes() + path.unlink() + try: + os.symlink(private, path) + except (OSError, NotImplementedError) as error: + path.write_bytes(original) + self.skipTest(f'symlinks are not available here: {error}') + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked ' + name + ' must be refused') + self.assertIn('REFUSED: symlinked source', output) + self.assertIn(name + ' (symlink)', output) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + path.unlink() + path.write_bytes(original) + result = self.wheel() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_trailing_separator_or_parent_step_does_not_hide_a_symlink(self): + # ``bl/`` and ``bl/.`` name whatever ``bl`` points at, and ``x/..`` + # leaves the target of a symlinked ``x``; the check must see the link + # itself in each spelling, as it does for the plain name. + outside = self.base / 'outside_lib' + victim = outside / PACKAGE / '__init__.py' + victim.parent.mkdir(parents=True) + victim.write_text('VICTIM = True\n', encoding='utf-8') + deep = self.base / 'deep' / 'inner' + deep.mkdir(parents=True) + try: + os.symlink(outside, self.checkout / 'bl', target_is_directory=True) + os.symlink(deep, self.checkout / 'x', target_is_directory=True) + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + for spelling, link in (('bl', 'bl'), ('bl/', 'bl'), ('bl/.', 'bl'), ('x/../escaped', 'x')): + with self.subTest(spelling=spelling): + result = direct_setup(self.checkout, 'build_py', '--build-lib', spelling) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, spelling + ' must be refused') + self.assertIn(link + ' (symlink)', output) + self.assertEqual(victim.read_text(encoding='utf-8'), 'VICTIM = True\n') + self.assertFalse((self.base / 'deep' / 'escaped').exists(), 'nothing may be written through x') + result = direct_setup(self.checkout, 'bdist_wheel', '--bdist-dir', 'bl/', '--dist-dir', str(self.wheels)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a staging directory named with a trailing separator is checked too') + self.assertIn('bl (symlink)', output) + self.assertEqual(sorted(path.name for path in outside.iterdir()), [PACKAGE]) + + def test_install_lib_admits_bytecode_only_in_the_tree_this_run_recompiled(self): + # ``build_py --compile`` rewrites bytecode under build/lib only; a cache + # file in another --build-dir was compiled by nobody in this run, and + # install_lib keeps a destination cache file that is newer than its source. + built = direct_setup(self.checkout, 'build_py') + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + elsewhere = self.checkout / 'elsewhere' + shutil.copytree(self.checkout / 'build' / 'lib', elsewhere) + cache = Path(importlib.util.cache_from_source(str(elsewhere / PACKAGE / '__init__.py'), optimization='')) + cache.parent.mkdir(exist_ok=True) + cache.write_bytes(b'not bytecode this run compiled') + result = direct_build_py(self.checkout, '--compile', 'install_lib', '--build-dir', 'elsewhere', + '--install-dir', str(self.target)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'bytecode in a tree this run did not recompile must be refused') + self.assertIn('REFUSED: stale build output', output) + self.assertIn(os.path.relpath(cache, self.checkout).replace(os.sep, '/'), output) + self.assertEqual(installed_files(self.target), []) + # From build/lib, which the same run has just recompiled, it is accepted. + result = direct_build_py(self.checkout, '--compile', 'install_lib', '--build-dir', 'build/lib', + '--install-dir', str(self.target)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(installed_files(self.target), tracked_source_files()) + + def test_editable_wheel_validates_a_pre_generated_dist_info_and_builds_quietly(self): + # pip's editable flow passes the metadata it generated to + # ``editable_wheel --dist-info-dir``, which copies it into the wheel as + # is. editable_wheel runs build_py in editable mode, where nothing is + # copied, so there is no build/lib to guard and nothing is reported. + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'editable_wheel', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, output) + self.assertNotIn('REFUSED', output) + self.assertEqual(len(sorted(dist.glob('*.whl'))), 1) + meta = self.base / 'meta' + meta.mkdir() + generated = direct_setup(self.checkout, 'dist_info', '--output-dir', str(meta)) + self.assertEqual(generated.returncode, 0, generated.stdout + generated.stderr) + dist_info = sorted(meta.glob('*.dist-info'))[0] + planted = dist_info / 'private.txt' + planted.write_text('private\n', encoding='utf-8') + editable_dist = self.base / 'editable_dist' + result = direct_setup(self.checkout, 'editable_wheel', '--dist-info-dir', str(dist_info), + '--dist-dir', str(editable_dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a pre-generated dist-info with an extra file must be refused') + self.assertIn('REFUSED: pre-generated metadata', output) + self.assertIn(os.path.relpath(planted, self.checkout).replace(os.sep, '/') + + " (not in this build's metadata)", output) + self.assertEqual(sorted(editable_dist.glob('*.whl')) if editable_dist.exists() else [], []) + planted.unlink() + result = direct_setup(self.checkout, 'editable_wheel', '--dist-info-dir', str(dist_info), + '--dist-dir', str(editable_dist)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(sorted(editable_dist.glob('*.whl'))), 1) + + def test_bdist_with_several_dumb_formats_accepts_its_own_kept_staging_tree(self): + # ``bdist`` keeps the staging tree between two formats of the same + # command; that tree is this run's own output, not a leftover. A tree + # kept by an earlier invocation is still refused. + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'bdist', '--formats=zip,gztar', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, output) + self.assertEqual(sorted(path.suffix for path in dist.iterdir()), ['.gz', '.zip']) + result = direct_setup(self.checkout, 'bdist_dumb', '--keep-temp', '--dist-dir', str(dist)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + result = direct_setup(self.checkout, 'bdist_dumb', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a tree kept by an earlier invocation is a leftover') + self.assertIn('REFUSED: stale build output', output) + @unittest.skipUnless(os.environ.get('QUERY_INSTALL_ISOLATED') == '1', 'isolated build downloads setuptools; set QUERY_INSTALL_ISOLATED=1 to run') def test_isolated_build_also_refuses_stale_build_lib_leftovers(self): From 5ffbc028edbaff8aea80ec1de630bd256426e480 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 16:50:08 +0000 Subject: [PATCH 12/14] Keep kept trees per command, refuse special files and a symlinked build/ for editable wheels Scientific effect: NONE. Fixes from the author-side read of 9b3d500b: - A staging tree kept earlier in the same run is admitted only to the command that kept it, and a spec-only bdist_rpm run keeps nothing. Before this, `bdist_rpm --spec-only --keep-temp` vouched for a planted spec and archive that a later bdist_wheel then shipped. - Every tree walk refuses a file that is not a regular file. A named pipe at a name bdist_rpm admits made the spec write hang. - editable_wheel refuses a symlinked build/. A strict editable build empties its link tree there, and setuptools would empty the link's target. - bdist_rpm admits the icon that every full run copies again, so a repeat run with --icon is no longer refused. - The dist-info refusal names each entry once. Inputs an option names explicitly, such as --icon, and rpmbuild's own tree under --no-rpm3-mode, are taken as given. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 18 ++++--- setup.py | 85 +++++++++++++++++++------------- tests/test_install_provenance.py | 84 +++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 8fcd712..f01545c 100644 --- a/README.md +++ b/README.md @@ -63,11 +63,13 @@ run that does not recompile it, such as `--skip-build` or a build under `python -B`, is refused. Every `bdist` command (`bdist_wheel`, `bdist_egg`, `bdist_dumb`, `bdist_rpm`) refuses a leftover in its own staging tree under `build/bdist.*/`, which it archives as found (an interrupted build leaves -one). It does not refuse a tree kept by an earlier command of the same run, as -`bdist` does between two formats, because that tree is the run's own output. -For `bdist_rpm`, the spec and source archive that every run rewrites, and a +one). It does not refuse a tree that the same command kept earlier in the +same run, as `bdist` does between two formats, because that tree is the +command's own output; a spec-only `bdist_rpm` keeps nothing. For `bdist_rpm`, +the spec, source archive and icon that every full run rewrites, and a successful run leaves behind, are not leftovers. A sibling command's tree is -never read. The wheel and editable-wheel commands also refuse a +never read, and a named pipe or other special file in any tree is refused. The +wheel and editable-wheel commands also refuse a pre-generated `--dist-info-dir` that differs in any name or byte from this build's own metadata, which a separate `setup.py dist_info` run regenerates for the comparison (`REFUSED: pre-generated metadata …`), and the wheel @@ -95,9 +97,11 @@ with another path, so the tree is not this build's output; the build copy itself unlinks its destination before writing and does not write through. A Windows junction counts as a symlink throughout. A `--dry-run` build (an option setuptools 81 removed) writes nothing and is not refused for that; leftovers -are still refused. An editable -install copies nothing and runs from `src/`. The output directory (`dist/` or -`--dist-dir`) is taken as given. +are still refused. An editable install copies nothing and runs from `src/`; +the editable-wheel command still refuses a symlinked `build/`, where a strict +editable build keeps its link tree. The output directory (`dist/` or +`--dist-dir`) is taken as given, as are inputs an option names explicitly, +such as `bdist_rpm --icon`. `pyproject.toml` also declares that the package ships no package data. Nothing is deleted for you; remove the leftovers before installing: `git clean -fdX -- build src/universal_law_query.egg-info`. diff --git a/setup.py b/setup.py index c17877b..f0ee855 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,8 @@ and bytecode, which nothing recompiles, is refused. - ``editable_wheel`` applies the same ``--dist-info-dir`` comparison; pip's editable install passes the metadata it generated, and the command copies it - into the editable wheel as is. + into the editable wheel as is. It also refuses a symlinked ``build/``, where + a strict editable build empties and fills its link tree. - ``install_lib`` applies the same comparison to the directory it copies from (its ``--build-dir``, by default ``build/lib``), whether or not it rebuilt it, which covers the legacy ``install``, ``install_lib`` and ``bdist_egg`` @@ -49,12 +50,14 @@ nothing recompiled it, and the copy keeps a cache file newer than its source. - ``bdist_egg`` and ``bdist_dumb`` refuse a leftover in their own staging trees the same way; the legacy archives are assembled from whatever those - trees hold. ``bdist_rpm`` refuses one in its rpm tree, except the spec and - source archive that every run rewrites (forced) and a successful run leaves - behind with empty directories. Each command reads only its own tree, so a - sibling's tree under ``build/bdist.`` blocks nothing else. A tree - that an earlier command of the same run kept (``bdist`` keeps one between - two formats of a command) is that run's own output and is admitted. + trees hold. ``bdist_rpm`` refuses one in its rpm tree, except the spec, + source archive and icon that every full run rewrites (forced) and a + successful run leaves behind with empty directories. Each command reads only + its own tree, so a sibling's tree under ``build/bdist.`` blocks + nothing else. A tree that the same command kept earlier in the same run + (``bdist`` keeps one between two formats of a command) is that command's own + output and is admitted to it alone; a spec-only rpm run keeps nothing. Every + tree walk refuses a file that is not a regular file, such as a named pipe. - ``egg_info`` regenerates ``SOURCES.txt`` from the tree and never reads a leftover manifest back in, so a stale entry can add a file neither to a wheel nor to a source distribution; any other file left beside the ones this run @@ -80,7 +83,9 @@ anything is written, because every write under it would land wherever the link points. A tree named with a trailing separator or ``.`` is checked as the link it names, and one reached through ``..`` after a symlinked directory is refused -as well. The output directory (``dist/`` or ``--dist-dir``) is taken as given. +as well. The output directory (``dist/`` or ``--dist-dir``) is taken as given, +as are inputs an option names explicitly, such as ``bdist_rpm --icon``, and +the tree ``rpmbuild`` uses under ``--no-rpm3-mode``. Nothing is deleted; each refusal names the leftovers and the remedy, which is ``git clean`` for the checkout's own ``build/`` and egg-info directory and @@ -177,6 +182,8 @@ def _unexpected(tree: str, produced: set[str], empty_dirs: bool = False) -> list kind = _alias(path) if kind: found.append(_relative(path) + ' (' + kind + ')') + elif not os.path.isfile(path): + found.append(_relative(path) + ' (not a regular file)') # a pipe would block a reader or writer elif os.stat(path).st_nlink > 1: found.append(_relative(path) + ' (hard link)') elif _canonical(path) not in produced: @@ -412,7 +419,7 @@ def _dist_info_mismatches(tree: str, reference: str) -> list[str]: def _refuse_dist_info(mismatches: list[str]) -> None: if mismatches: raise SetupError( - 'REFUSED: pre-generated metadata does not match this build\'s own: ' + ', '.join(mismatches) + 'REFUSED: pre-generated metadata does not match this build\'s own: ' + ', '.join(sorted(set(mismatches))) + '. Regenerate it from this checkout (setup.py dist_info) or build without --dist-info-dir.') @@ -425,28 +432,31 @@ def _guard_dist_info(dist_info_dir: str | None) -> None: _refuse_dist_info(_dist_info_mismatches(os.fspath(dist_info_dir), _fresh_dist_info(workdir))) -# Files of staging trees that an earlier command of this same run kept -# (``bdist`` keeps the tree between two formats of one command): this run's -# own output, not leftovers. -_KEPT_STAGING: set[str] = set() +# Files of a staging tree that a command kept earlier in this same run, by +# command (``bdist`` keeps the tree between two formats of one command): that +# command's own output, not leftovers. Another command does not archive it. +_KEPT_STAGING: dict[str, set[str]] = {} -def _guard_staging(staging_dir: str, rewritten: frozenset[str] = frozenset(), empty_dirs: bool = True) -> None: +def _guard_staging(command, staging_dir: str, rewritten: frozenset[str] = frozenset(), + empty_dirs: bool = True) -> None: """Refuse any leftover in a command's own staging tree, which the command archives (or, for rpm, hands to rpmbuild) as found. A sibling command's - tree under the same ``build/bdist.`` is never read. Files that an - earlier command of this same run kept, and ``rewritten`` files that the - command overwrites on every run, are admitted.""" + tree under the same ``build/bdist.`` is never read. Files that + the same command kept earlier in this run, and ``rewritten`` files that it + overwrites on every run, are admitted.""" + kept = _KEPT_STAGING.get(command.get_command_name(), set()) _refuse(sorted(_symlinked_parents(staging_dir) - + _unexpected(staging_dir, _KEPT_STAGING | rewritten, empty_dirs=empty_dirs)), staging_dir) + + _unexpected(staging_dir, kept | rewritten, empty_dirs=empty_dirs)), staging_dir) def _remember_kept(command, staging_dir: str) -> None: - """After a command that keeps its staging tree, admit that tree's files to - later commands of this run.""" + """After a run that keeps its staging tree, admit that tree's files to the + same command's next run in this process.""" if getattr(command, 'keep_temp', False) and os.path.isdir(staging_dir): + kept = _KEPT_STAGING.setdefault(command.get_command_name(), set()) for dirpath, _dirnames, filenames in os.walk(staging_dir): - _KEPT_STAGING.update(_canonical(os.path.join(dirpath, name)) for name in filenames) + kept.update(_canonical(os.path.join(dirpath, name)) for name in filenames) class build_py(_build_py): @@ -471,7 +481,7 @@ class bdist_wheel(_bdist_wheel): """Refuse a leftover staging tree, and re-check build/lib even with --skip-build.""" def run(self) -> None: - _guard_staging(self.bdist_dir) + _guard_staging(self, self.bdist_dir) _guard_dist_info(self.dist_info_dir) if self.skip_build: _guard_skipped_build(self) @@ -487,7 +497,10 @@ class editable_wheel(_editable_wheel): def run(self) -> None: # Before the run: it writes WHEEL into the given directory, then copies - # the directory into the wheel as is. + # the directory into the wheel as is. A strict editable build empties + # and fills its link tree under build/, which a symlinked build/ would + # redirect. + _refuse(_symlinked_parents(os.path.join(self.project_dir, 'build', '__editable__')), 'build') _guard_dist_info(self.dist_info_dir) super().run() @@ -496,7 +509,7 @@ class bdist_egg(_bdist_egg): """Refuse a leftover egg staging tree; the legacy archive is assembled from whatever it holds.""" def run(self) -> None: - _guard_staging(self.bdist_dir) + _guard_staging(self, self.bdist_dir) super().run() _remember_kept(self, self.bdist_dir) @@ -505,7 +518,7 @@ class bdist_dumb(_bdist_dumb): """Refuse a leftover dumb staging tree; the archive is the pseudo-installation tree as found.""" def run(self) -> None: - _guard_staging(self.bdist_dir) + _guard_staging(self, self.bdist_dir) super().run() _remember_kept(self, self.bdist_dir) @@ -514,18 +527,22 @@ class bdist_rpm(_bdist_rpm): """Refuse a leftover in the rpm build tree before rpmbuild sees it.""" def run(self) -> None: - # Every run rewrites the spec and the source archive that rpmbuild - # reads, and a successful run leaves both behind with empty - # directories; the copy is forced, so an existing archive newer than - # the new one is not kept. Anything else in the tree is a leftover. + # Every full run rewrites the spec, the source archive and any icon + # that rpmbuild reads, and a successful run leaves them behind with + # empty directories; the copies are forced, so an existing file newer + # than the new one is not kept. Anything else in the tree is a leftover. self.force = True archive = self.distribution.get_fullname() + ('.tar.bz2' if self.use_bzip2 else '.tar.gz') - rewritten = frozenset({ - _canonical(os.path.join(self.rpm_base, 'SPECS', self.distribution.get_name() + '.spec')), - _canonical(os.path.join(self.rpm_base, 'SOURCES', archive))}) - _guard_staging(self.rpm_base, rewritten, empty_dirs=False) + names = [os.path.join('SPECS', self.distribution.get_name() + '.spec'), os.path.join('SOURCES', archive)] + if self.icon: + names.append(os.path.join('SOURCES', os.path.basename(self.icon))) + rewritten = frozenset(_canonical(os.path.join(self.rpm_base, name)) for name in names) + _guard_staging(self, self.rpm_base, rewritten, empty_dirs=False) super().run() - _remember_kept(self, self.rpm_base) + if not self.spec_only: + # A spec-only run writes the spec into --dist-dir and rewrites + # nothing in the rpm tree, so it vouches for nothing found there. + _remember_kept(self, self.rpm_base) class install_lib(_install_lib): diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index 3add5c1..5ccd776 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -369,6 +369,12 @@ def test_what_a_successful_bdist_rpm_leaves_is_not_a_leftover(self): result = direct_setup(self.checkout, 'bdist_rpm', '--spec-only', '--dist-dir', str(spec_dir)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(sorted(path.name for path in spec_dir.iterdir()), ['universal-law-query.spec']) + # The icon an earlier --icon run copied into SOURCES is rewritten too. + icon = self.base / 'icon.xpm' + icon.write_text('/* XPM */\n', encoding='utf-8') + (rpm / 'SOURCES' / 'icon.xpm').write_text('/* earlier XPM */\n', encoding='utf-8') + result = direct_setup(self.checkout, 'bdist_rpm', '--spec-only', '--icon', str(icon), '--dist-dir', str(spec_dir)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) stale = rpm / 'SOURCES' / 'private.txt' stale.write_text('PRIVATE\n', encoding='utf-8') result = direct_setup(self.checkout, 'bdist_rpm', '--spec-only', '--dist-dir', str(spec_dir)) @@ -376,6 +382,72 @@ def test_what_a_successful_bdist_rpm_leaves_is_not_a_leftover(self): self.assertNotEqual(result.returncode, 0, 'another file in the rpm tree is a leftover') self.assertIn(stale.relative_to(self.checkout).as_posix(), output) + def test_a_kept_tree_is_admitted_only_to_the_command_that_kept_it(self): + # bdist keeps a tree between two formats of one command. Another + # command in the same run does not archive it, and a spec-only rpm run + # rewrites nothing in its tree, so neither may pass on what it found. + staging = wheel_staging_dir(self.checkout) + planted = staging / 'SOURCES' / 'universal_law_query-0.1.0.tar.gz' + planted.parent.mkdir(parents=True) + planted.write_bytes(b'FOREIGN\n') + dist = self.base / 'dist' + dist.mkdir() + result = direct_setup(self.checkout, 'bdist_rpm', '--spec-only', '--keep-temp', + '--rpm-base', staging.relative_to(self.checkout).as_posix(), + '--dist-dir', str(dist), 'bdist_wheel', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a spec-only run may not vouch for what it found') + self.assertIn(planted.relative_to(self.checkout).as_posix(), output) + self.assertEqual(sorted(dist.glob('*.whl')), []) + shutil.rmtree(staging.parent) + result = direct_setup(self.checkout, 'bdist_dumb', '--keep-temp', '--bdist-dir', 'shared', + '--dist-dir', str(dist), 'bdist_wheel', '--bdist-dir', 'shared', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, "another command's kept tree is not this command's output") + self.assertIn('REFUSED: stale build output', output) + self.assertEqual(sorted(dist.glob('*.whl')), []) + + def test_non_regular_files_in_a_staging_tree_are_refused(self): + # A named pipe or other special file is not build output, and a write + # or archive step that opens it can hang or read another process's bytes. + if not hasattr(os, 'mkfifo'): + self.skipTest('named pipes are not available here') + fifo = wheel_staging_dir(self.checkout) / PACKAGE / 'pipe' + fifo.parent.mkdir(parents=True) + os.mkfifo(fifo) + result = direct_setup(self.checkout, 'bdist_wheel', '--dist-dir', str(self.wheels)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0) + self.assertIn(fifo.relative_to(self.checkout).as_posix() + ' (not a regular file)', output) + # Also at a name bdist_rpm admits because every run rewrites it: the + # spec write would block on the pipe. + spec = wheel_staging_dir(self.checkout).parent / 'rpm' / 'SPECS' / 'universal-law-query.spec' + spec.parent.mkdir(parents=True) + os.mkfifo(spec) + result = direct_setup(self.checkout, 'bdist_rpm', '--dist-dir', str(self.base / 'dist')) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0) + self.assertIn(spec.relative_to(self.checkout).as_posix() + ' (not a regular file)', output) + + def test_strict_editable_build_refuses_a_symlinked_build_directory(self): + # A strict editable install empties and fills its link tree under + # build/; through a symlinked build/ setuptools would empty the + # link's target. + outside = self.base / 'outside_build' + keep = outside / ('__editable__.' + PACKAGE + '-0.1.0-py3-none-any') / 'keepme.txt' + keep.parent.mkdir(parents=True) + keep.write_text('KEEP\n', encoding='utf-8') + try: + os.symlink(outside, self.checkout / 'build', target_is_directory=True) + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + dist = self.base / 'dist' + result = direct_setup(self.checkout, 'editable_wheel', '--mode', 'strict', '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked build/ must be refused for a strict editable build') + self.assertIn('build (symlink)', output) + self.assertEqual(keep.read_text(encoding='utf-8'), 'KEEP\n', 'nothing may be emptied through the link') + def test_windows_junctions_are_treated_as_symlinks(self): # A Windows junction is not reported by os.path.islink; Python 3.12+ # reports it through os.path.isjunction and every supported version @@ -495,6 +567,18 @@ def refused(change: str, path: Path, label: str) -> None: refused('a ' + kind + 'ed entry', top_level, ' (' + kind + ')') top_level.unlink() top_level.write_bytes(original[top_level]) + # The directory itself as a symlink, named with a trailing separator, + # is reported once. + try: + os.symlink(dist_info, self.base / 'linked.dist-info', target_is_directory=True) + except (OSError, NotImplementedError): + pass # not available on this filesystem + else: + linked = os.path.relpath(self.base / 'linked.dist-info', self.checkout) + result = direct_setup(self.checkout, 'bdist_wheel', '--dist-info-dir', linked + os.sep, '--dist-dir', str(dist)) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked dist-info directory must be refused') + self.assertEqual(output.count('linked.dist-info (symlink)'), 1, output) # The directory as generated is accepted, and the wheel carries exactly it. result = direct_setup(self.checkout, 'bdist_wheel', '--dist-info-dir', str(dist_info), '--dist-dir', str(dist)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) From 24bee65d53c64a43385bf3f6ba48bb777822b3aa Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 16:54:55 +0000 Subject: [PATCH 13/14] Admit pip's own metadata under a symlinked TMPDIR, a rewritten WHEEL, both rpm archives Scientific effect: NONE. Fixes from the author-side false-refusal read of 9b3d500b: - The manifest symlink check skips absolute entries. They are this run's own egg-info under a configured --egg-base, run() checks them, and they are taken as given above it. With a symlinked TMPDIR (the macOS default) the check walked pip's temporary path up to "/", and pip wheel and pip install refused every build. - A regular top-level WHEEL in a pre-generated dist-info is admitted. bdist_wheel and editable_wheel both rewrite it; editable_wheel writes it into the given directory, so reusing that directory was refused. Links and hard links are still refused first. - bdist_rpm admits the source archive of either compression. A run with the other --use-bzip2 setting leaves one that the spec never reads. - The docs no longer say that pip passes its metadata to editable_wheel. pip 26 with setuptools 77 to 84 does not. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- README.md | 3 ++- setup.py | 33 ++++++++++++++++--------- tests/test_install_provenance.py | 42 +++++++++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f01545c..a66a0a3 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,8 @@ never read, and a named pipe or other special file in any tree is refused. The wheel and editable-wheel commands also refuse a pre-generated `--dist-info-dir` that differs in any name or byte from this build's own metadata, which a separate `setup.py dist_info` run regenerates -for the comparison (`REFUSED: pre-generated metadata …`), and the wheel +for the comparison (`REFUSED: pre-generated metadata …`; a regular `WHEEL`, +which both commands rewrite, is admitted), and the wheel command repeats the `build/lib/` check, so `--skip-build` cannot bypass it; with `--skip-build` every expected output must be present with the bytes of its source. The legacy `install`, `install_lib` and `bdist_egg` commands run diff --git a/setup.py b/setup.py index f0ee855..ba60e26 100644 --- a/setup.py +++ b/setup.py @@ -34,14 +34,16 @@ pre-generated ``--dist-info-dir`` that differs in any name or byte from this build's own metadata, which a separate ``dist_info`` run of this script regenerates for the comparison (the directory is copied into the wheel as - is), and repeats the ``build/lib`` comparison, so ``--skip-build`` cannot + is; only a regular ``WHEEL``, which the command rewrites, is admitted + besides), and repeats the ``build/lib`` comparison, so ``--skip-build`` cannot bypass it. With ``--skip-build`` nothing is rebuilt, so the byte comparison runs there instead, every expected output must be present as a regular file, and bytecode, which nothing recompiles, is refused. -- ``editable_wheel`` applies the same ``--dist-info-dir`` comparison; pip's - editable install passes the metadata it generated, and the command copies it - into the editable wheel as is. It also refuses a symlinked ``build/``, where - a strict editable build empties and fills its link tree. +- ``editable_wheel`` applies the same ``--dist-info-dir`` comparison; a + frontend may pass the metadata it generated (pip 26 with setuptools 77 to 84 + does not), and the command writes ``WHEEL`` into it and copies it into the + editable wheel as is. It also refuses a symlinked ``build/``, where a strict + editable build empties and fills its link tree. - ``install_lib`` applies the same comparison to the directory it copies from (its ``--build-dir``, by default ``build/lib``), whether or not it rebuilt it, which covers the legacy ``install``, ``install_lib`` and ``bdist_egg`` @@ -382,7 +384,9 @@ def _dist_info_mismatches(tree: str, reference: str) -> list[str]: """Every way a pre-generated dist-info directory differs from ``reference``, this build's own: an alias or hard link, a non-regular file, an entry the reference lacks, bytes that differ, or a reference file that is missing. - Names are compared exactly, so a case variant counts as another entry.""" + Names are compared exactly, so a case variant counts as another entry. A + regular top-level ``WHEEL`` is admitted: bdist_wheel and editable_wheel + both rewrite it.""" found = _symlinked_parents(tree) kind = _alias(tree) if kind: @@ -407,6 +411,8 @@ def _dist_info_mismatches(tree: str, reference: str) -> list[str]: found.append(_relative(path) + ' (not a regular file)') elif os.stat(path).st_nlink > 1: found.append(_relative(path) + ' (hard link)') + elif entry == 'WHEEL': + pass # bdist_wheel and editable_wheel rewrite it (editable_wheel in place) elif entry not in expected_files: found.append(_relative(path) + ' (not in this build\'s metadata)') elif not filecmp.cmp(os.path.join(reference, entry), path, shallow=False): @@ -493,7 +499,7 @@ def run(self) -> None: class editable_wheel(_editable_wheel): - """Validate a pre-generated dist-info; pip's editable flow passes the one it generated.""" + """Validate a pre-generated dist-info, which a frontend may pass, and a symlinked build/.""" def run(self) -> None: # Before the run: it writes WHEEL into the given directory, then copies @@ -532,8 +538,10 @@ def run(self) -> None: # empty directories; the copies are forced, so an existing file newer # than the new one is not kept. Anything else in the tree is a leftover. self.force = True - archive = self.distribution.get_fullname() + ('.tar.bz2' if self.use_bzip2 else '.tar.gz') - names = [os.path.join('SPECS', self.distribution.get_name() + '.spec'), os.path.join('SOURCES', archive)] + # Both archive formats: a run with the other --use-bzip2 setting leaves + # one that the spec, naming only the current one, never reads. + names = [os.path.join('SPECS', self.distribution.get_name() + '.spec')] + names += [os.path.join('SOURCES', self.distribution.get_fullname() + suffix) for suffix in ('.tar.gz', '.tar.bz2')] if self.icon: names.append(os.path.join('SOURCES', os.path.basename(self.icon))) rewritten = frozenset(_canonical(os.path.join(self.rpm_base, name)) for name in names) @@ -577,8 +585,11 @@ def find_sources(self) -> None: self.filelist = mm.filelist # The metadata conversion reads README.md into METADATA and copies the # license files into the wheel, following any symlink, so every file - # the manifest names must be a file of this tree, as for an sdist. - _refuse_sources(_symlinked_paths(self.filelist.files)) + # the manifest names must be a file of this tree, as for an sdist. An + # absolute entry is this run's own egg-info under a configured + # --egg-base (pip's temporary directory), checked by run() and taken + # as given above it. + _refuse_sources(_symlinked_paths(name for name in self.filelist.files if not os.path.isabs(name))) def run(self) -> None: written = {entry.name for entry in _metadata.entry_points(group='egg_info.writers')} diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index 5ccd776..7b0e4bb 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -363,6 +363,8 @@ def test_what_a_successful_bdist_rpm_leaves_is_not_a_leftover(self): (rpm / name).mkdir(parents=True) (rpm / 'SPECS' / 'universal-law-query.spec').write_text('%define name universal-law-query\n', encoding='utf-8') (rpm / 'SOURCES' / 'universal_law_query-0.1.0.tar.gz').write_bytes(b'an earlier source archive') + # A run with the other --use-bzip2 setting left its archive too. + (rpm / 'SOURCES' / 'universal_law_query-0.1.0.tar.bz2').write_bytes(b'an earlier bzip2 archive') result = self.wheel() self.assertEqual(result.returncode, 0, result.stdout + result.stderr) spec_dir = self.base / 'spec' @@ -539,11 +541,15 @@ def refused(change: str, path: Path, label: str) -> None: refused('a case variant of a file', variant, " (not in this build's metadata)") variant.unlink() top_level.write_bytes(original[top_level]) - # bdist_wheel writes WHEEL and RECORD itself; the conversion produces neither. + # bdist_wheel and editable_wheel write WHEEL themselves (editable_wheel + # into the given directory), so a regular WHEEL is rewritten, not + # shipped; RECORD, which neither writes there, is not admitted. + record = dist_info / 'RECORD' + record.write_text('private,,\n', encoding='utf-8') + refused('a RECORD file', record, " (not in this build's metadata)") + record.unlink() wheel_file = dist_info / 'WHEEL' - wheel_file.write_text('Wheel-Version: 1.0\n', encoding='utf-8') - refused('a WHEEL file', wheel_file, " (not in this build's metadata)") - wheel_file.unlink() + wheel_file.write_text('Wheel-Version: 1.0\nGenerator: not this build\n', encoding='utf-8') # Under licenses/ only the declared license files belong, and no other # directory belongs anywhere. stray_license = dist_info / 'licenses' / 'private.txt' @@ -589,9 +595,11 @@ def refused(change: str, path: Path, label: str) -> None: shipped = sorted(name.split('/', 1)[1] for name in archive.namelist() if name.startswith(dist_info.name + '/')) shipped_metadata = archive.read(dist_info.name + '/METADATA') + shipped_wheel = archive.read(dist_info.name + '/WHEEL') self.assertNotIn('private.txt', shipped) self.assertIn('METADATA', shipped) self.assertEqual(shipped_metadata, original[metadata]) + self.assertNotIn(b'not this build', shipped_wheel, 'the planted WHEEL must be rewritten') def test_hard_linked_build_outputs_are_refused(self): # A hard link at a produced path shares its bytes with another file. @@ -1186,6 +1194,11 @@ def test_editable_wheel_validates_a_pre_generated_dist_info_and_builds_quietly(s '--dist-dir', str(editable_dist)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(len(sorted(editable_dist.glob('*.whl'))), 1) + # editable_wheel wrote WHEEL into the directory; using it again is accepted. + self.assertTrue((dist_info / 'WHEEL').is_file()) + result = direct_setup(self.checkout, 'editable_wheel', '--dist-info-dir', str(dist_info), + '--dist-dir', str(editable_dist)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) def test_bdist_with_several_dumb_formats_accepts_its_own_kept_staging_tree(self): # ``bdist`` keeps the staging tree between two formats of the same @@ -1203,6 +1216,27 @@ def test_bdist_with_several_dumb_formats_accepts_its_own_kept_staging_tree(self) self.assertNotEqual(result.returncode, 0, 'a tree kept by an earlier invocation is a leftover') self.assertIn('REFUSED: stale build output', output) + def test_a_symlinked_temporary_directory_is_not_refused(self): + # pip builds the metadata under TMPDIR, an absolute path that is taken + # as given; macOS's default temporary directory sits behind a symlink. + real = self.base / 'real_tmp' + real.mkdir() + try: + os.symlink(real, self.base / 'linked_tmp', target_is_directory=True) + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + env = dict(os.environ) + env.update(TMPDIR=str(self.base / 'linked_tmp'), PIP_DISABLE_PIP_VERSION_CHECK='1', PYTHONDONTWRITEBYTECODE='1') + result = subprocess.run([sys.executable, '-m', 'pip', 'wheel', '--quiet', '--no-deps', '--no-build-isolation', + '--wheel-dir', str(self.wheels), str(self.checkout)], + env=env, capture_output=True, text=True, timeout=PIP_TIMEOUT) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(sorted(self.wheels.glob('*.whl'))), 1) + meta = self.base / 'linked_tmp' / 'meta' + meta.mkdir() + result = direct_setup(self.checkout, 'dist_info', '--output-dir', str(meta)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + @unittest.skipUnless(os.environ.get('QUERY_INSTALL_ISOLATED') == '1', 'isolated build downloads setuptools; set QUERY_INSTALL_ISOLATED=1 to run') def test_isolated_build_also_refuses_stale_build_lib_leftovers(self): From 72c3e3f9ce4ec67bf8d881b15fb3d0d9bfe37152 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 17:19:10 +0000 Subject: [PATCH 14/14] Add controls for protections that no test caught yet Scientific effect: NONE. Tests only. The author-side mutation read of 9b3d500b found protections that a mutant could remove without any test failing. Each new assertion fails against its mutant and passes at head: - bdist_rpm's forced copy (m8): a full bdist_rpm run with stand-in rpm and rpmbuild scripts written into the test's temporary directory replaces a future-dated leftover source archive. - install_lib --skip-build (m22): with setup.cfg [build_py] compile = 1, a cache file that nothing recompiled is refused. - The junction prune in both walks (m11, m11b): a file behind a simulated junction is never reported. - A symlinked directory above a manifest file (m14): a symlinked tests/ directory is refused for pip wheel. - An absolute 'bl/' spelling (m2); each leftover is named once (m13); setuptools' own bdist_rpm spec line (m10). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XjmjBhHbkztFy4aSubbtiS --- tests/test_install_provenance.py | 105 ++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 3 deletions(-) diff --git a/tests/test_install_provenance.py b/tests/test_install_provenance.py index 7b0e4bb..a742a9e 100644 --- a/tests/test_install_provenance.py +++ b/tests/test_install_provenance.py @@ -371,6 +371,9 @@ def test_what_a_successful_bdist_rpm_leaves_is_not_a_leftover(self): result = direct_setup(self.checkout, 'bdist_rpm', '--spec-only', '--dist-dir', str(spec_dir)) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(sorted(path.name for path in spec_dir.iterdir()), ['universal-law-query.spec']) + # setuptools' own bdist_rpm, which the guard extends, writes this install line. + self.assertIn('setup.py install --single-version-externally-managed', + (spec_dir / 'universal-law-query.spec').read_text(encoding='utf-8')) # The icon an earlier --icon run copied into SOURCES is rewritten too. icon = self.base / 'icon.xpm' icon.write_text('/* XPM */\n', encoding='utf-8') @@ -459,6 +462,9 @@ def test_windows_junctions_are_treated_as_symlinks(self): # would otherwise write every module into. junction = self.checkout / 'build' / 'lib' / PACKAGE junction.mkdir(parents=True) + # A file behind the junction: no walk may descend into it. + beyond = junction / 'beyond.txt' + beyond.write_text('beyond the junction\n', encoding='utf-8') target = os.path.normcase(os.path.abspath(junction)) via_isjunction = ('import os\n_t = ' + repr(target) + '\n' 'os.path.isjunction = lambda p: os.path.normcase(os.path.abspath(p)) == _t\n') @@ -484,7 +490,8 @@ def test_windows_junctions_are_treated_as_symlinks(self): self.assertNotEqual(result.returncode, 0, 'a junction must be refused like a symlink') self.assertIn('REFUSED: stale build output', output) self.assertIn('build/lib/' + PACKAGE + ' (junction)', output) - self.assertEqual(sorted(junction.iterdir()), [], 'nothing may be written through the junction') + self.assertNotIn('beyond.txt', output, 'no walk may descend into a junction') + self.assertEqual(sorted(junction.iterdir()), [beyond], 'nothing may be written through the junction') # A junction among the package sources is refused before any copy. shutil.rmtree(self.checkout / 'build') source = os.path.normcase(os.path.abspath(self.checkout / SOURCE_DIR)) @@ -1124,12 +1131,13 @@ def test_a_trailing_separator_or_parent_step_does_not_hide_a_symlink(self): os.symlink(deep, self.checkout / 'x', target_is_directory=True) except (OSError, NotImplementedError) as error: self.skipTest(f'symlinks are not available here: {error}') - for spelling, link in (('bl', 'bl'), ('bl/', 'bl'), ('bl/.', 'bl'), ('x/../escaped', 'x')): + for spelling, link in (('bl', 'bl'), ('bl/', 'bl'), ('bl/.', 'bl'), ('x/../escaped', 'x'), + (str(self.checkout / 'bl') + os.sep, 'bl')): with self.subTest(spelling=spelling): result = direct_setup(self.checkout, 'build_py', '--build-lib', spelling) output = result.stdout + result.stderr self.assertNotEqual(result.returncode, 0, spelling + ' must be refused') - self.assertIn(link + ' (symlink)', output) + self.assertEqual(output.count(link + ' (symlink)'), 1, 'each leftover is named once: ' + output) self.assertEqual(victim.read_text(encoding='utf-8'), 'VICTIM = True\n') self.assertFalse((self.base / 'deep' / 'escaped').exists(), 'nothing may be written through x') result = direct_setup(self.checkout, 'bdist_wheel', '--bdist-dir', 'bl/', '--dist-dir', str(self.wheels)) @@ -1216,6 +1224,97 @@ def test_bdist_with_several_dumb_formats_accepts_its_own_kept_staging_tree(self) self.assertNotEqual(result.returncode, 0, 'a tree kept by an earlier invocation is a leftover') self.assertIn('REFUSED: stale build output', output) + def test_symlinked_directory_above_a_manifest_file_is_refused_for_wheels(self): + # The manifest names tests/ too; a symlinked directory above any of + # its files would put the link's target into the metadata build. + moved = self.base / 'tests_elsewhere' + shutil.move(str(self.checkout / 'tests'), str(moved)) + try: + os.symlink(moved, self.checkout / 'tests', target_is_directory=True) + except (OSError, NotImplementedError) as error: + self.skipTest(f'symlinks are not available here: {error}') + result = self.wheel() + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a symlinked directory above a manifest file must be refused') + self.assertIn('REFUSED: symlinked source', output) + self.assertIn('tests (symlink)', output) + self.assertEqual(sorted(self.wheels.glob('*.whl')), []) + + def test_dist_info_comparison_does_not_descend_into_a_junction(self): + meta = self.base / 'meta' + meta.mkdir() + generated = direct_setup(self.checkout, 'dist_info', '--output-dir', str(meta)) + self.assertEqual(generated.returncode, 0, generated.stdout + generated.stderr) + dist_info = sorted(meta.glob('*.dist-info'))[0] + licenses = dist_info / 'licenses' + (licenses / 'beyond.txt').write_text('beyond the junction\n', encoding='utf-8') + target = os.path.normcase(os.path.abspath(licenses)) + preamble = ('import os\n_t = ' + repr(target) + '\n' + 'os.path.isjunction = lambda p: os.path.normcase(os.path.abspath(p)) == _t\n') + result = setup_with_preamble(self.checkout, preamble, 'bdist_wheel', '--dist-info-dir', str(dist_info), + '--dist-dir', str(self.base / 'dist')) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'a junction in a pre-generated dist-info must be refused') + self.assertIn(os.path.relpath(licenses, self.checkout).replace(os.sep, '/') + ' (junction)', output) + self.assertNotIn('beyond.txt', output, 'no walk may descend into a junction') + + @unittest.skipUnless(os.name == 'posix', 'the stand-in rpm tools are shell scripts') + def test_bdist_rpm_rewrites_a_newer_source_archive_it_admits(self): + # The rpm tree's source archive is admitted by name because every run + # copies it again. The copy is forced, so a leftover newer than the + # fresh sdist is replaced rather than handed to rpmbuild. Stand-in rpm + # and rpmbuild scripts take the place of the real tools. + rpm = wheel_staging_dir(self.checkout).parent / 'rpm' + archive = rpm / 'SOURCES' / 'universal_law_query-0.1.0.tar.gz' + archive.parent.mkdir(parents=True) + archive.write_bytes(b'an earlier source archive') + future = os.stat(self.checkout / 'setup.py').st_mtime + 365 * 86400 + os.utime(archive, (future, future)) + tools = self.base / 'rpmtools' + tools.mkdir() + (tools / 'rpm').write_text( + '#!/bin/sh\necho "universal-law-query-0.1.0-1.src.rpm noarch/universal-law-query-0.1.0-1.noarch.rpm"\n', + encoding='utf-8') + (tools / 'rpmbuild').write_text( + '#!/bin/sh\ntop=""\nwhile [ $# -gt 0 ]; do case "$1" in --define) shift; case "$1" in ' + '_topdir*) top="${1#_topdir }";; esac;; esac; shift; done\n' + 'mkdir -p "$top/RPMS/noarch" "$top/SRPMS"\n' + 'echo stand-in > "$top/SRPMS/universal-law-query-0.1.0-1.src.rpm"\n' + 'echo stand-in > "$top/RPMS/noarch/universal-law-query-0.1.0-1.noarch.rpm"\n', encoding='utf-8') + for tool in ('rpm', 'rpmbuild'): + os.chmod(tools / tool, 0o755) + env = dict(os.environ) + env['PYTHONDONTWRITEBYTECODE'] = '1' + env['PATH'] = str(tools) + os.pathsep + env.get('PATH', '') + dist = self.base / 'dist' + dist.mkdir() + result = subprocess.run([sys.executable, 'setup.py', 'bdist_rpm', '--dist-dir', str(dist)], cwd=self.checkout, + env=env, capture_output=True, text=True, timeout=PIP_TIMEOUT) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotEqual(archive.read_bytes(), b'an earlier source archive', 'the leftover archive must be replaced') + with tarfile.open(archive) as fresh: + self.assertIn('universal_law_query-0.1.0/setup.py', fresh.getnames()) + + def test_install_lib_skip_build_refuses_bytecode_nothing_recompiled(self): + # With compilation configured, build_py writes bytecode; a later + # install_lib --skip-build recompiles nothing, so a cache file there is + # not this run's output, and the copy would keep a newer one. + (self.checkout / 'setup.cfg').write_text('[build_py]\ncompile = 1\n', encoding='utf-8') + built = direct_build_py(self.checkout) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + cache = Path(importlib.util.cache_from_source( + str(self.checkout / 'build' / 'lib' / PACKAGE / '__init__.py'), optimization='')) + self.assertTrue(cache.is_file(), 'the configured compile step must have written bytecode') + cache.write_bytes(b'PLANTED BYTECODE') + env = {key: value for key, value in os.environ.items() if key != 'PYTHONDONTWRITEBYTECODE'} + result = subprocess.run([sys.executable, 'setup.py', 'install_lib', '--skip-build', '--install-dir', + str(self.target)], cwd=self.checkout, env=env, capture_output=True, text=True, + timeout=PIP_TIMEOUT) + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, 'bytecode nothing recompiled must be refused') + self.assertIn(cache.relative_to(self.checkout).as_posix(), output) + self.assertEqual(installed_files(self.target), []) + def test_a_symlinked_temporary_directory_is_not_refused(self): # pip builds the metadata under TMPDIR, an absolute path that is taken # as given; macOS's default temporary directory sits behind a symlink.