From 45069fc44dae9073dee6e7825480822b2ee792a2 Mon Sep 17 00:00:00 2001 From: 8Dionysus Date: Mon, 7 Sep 2026 16:18:40 -0600 Subject: [PATCH] perf(validation): reuse checked-hash focused pytest cache --- VALIDATION.md | 42 ++- .../validation/validation_evidence_graph.json | 11 +- docs/validation/validation_lanes.json | 3 +- scripts/pytest_checked_hash_core.py | 273 ++++++++++++++++++ tests/test_pytest_checked_hash_core.py | 222 ++++++++++++++ 5 files changed, 539 insertions(+), 12 deletions(-) create mode 100644 scripts/pytest_checked_hash_core.py create mode 100644 tests/test_pytest_checked_hash_core.py diff --git a/VALIDATION.md b/VALIDATION.md index 70da6b24..442d152d 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -59,17 +59,43 @@ env -u PYTHONDONTWRITEBYTECODE PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 PYTHONPYCACHEPRE -k 'generation_identity or loaded_producer_source' ``` +For repeated local feedback on one focused core, the opt-in checked-hash +runner keeps one dedicated external prefix shared by the three focused cores. +Invoke it with `python3 -B`: + +```bash +python3 -B scripts/pytest_checked_hash_core.py --core privacy +python3 -B scripts/pytest_checked_hash_core.py --core outbox +python3 -B scripts/pytest_checked_hash_core.py --core import +``` + +Each invocation runs exactly one focused test through `pytest.main()` in the +initializer process. On the first invocation, an empty prefix is bound before +pytest is imported; after the test, the runner primes checked-hash bytecode +from sources in `sys.modules` plus the selected product-core source and the +selected test, using Python's bytecode API. The shared prefix is therefore +safe across later core selections even when their source has not been primed: +Python validates an available cache per file and otherwise falls back to +source. Later invocations only run the focused test. `-B` (also enforced +in-process) preserves +pytest assertion rewriting for ordinary failure diagnostics while preventing +a persistent `*-pytest-*.pyc` cache. The runner rejects a prefix containing +timestamp-based or pytest-rewrite bytecode, so do not point it at an ordinary +or previously reused prefix. It disables pytest's result cache and creates +only a local external bytecode cache; it does not install a runtime, change +global configuration, or write a receipt or registry. This is an owner-local +feedback route, not CI, release, installed-protocol, or runtime acceptance +evidence. + The real portable CLI/copy/install checks and the full source suite remain separate integration gates. -The bytecode prefix must remain outside the checkout. Pytest assertion -rewriting remains enabled for diagnostics. Python's default timestamp/size -invalidation normally recompiles when a source or test byte length or recorded -timestamp changes. The standard `.pyc` timestamp has one-second precision, so a -rapid same-size edit within the same timestamp second can reuse stale external -bytecode even when `st_mtime_ns` changes; preserving both stored fields has the -same limit. Rotate or clear the prefix when metadata-preserving or rapid -same-second edits are possible. CI's `runner.temp` prefix is fresh per job. +The checked-hash bytecode prefix must remain outside the checkout. Pytest +assertion rewriting remains enabled for diagnostics. The ordinary direct +commands above still use Python's default timestamp/size invalidation; a rapid +same-size edit within one timestamp second can reuse stale bytecode there, so +use a fresh prefix for those commands when needed. The checked-hash runner +validates source contents per file. CI's `runner.temp` prefix is fresh per job. ## Decisions diff --git a/docs/validation/validation_evidence_graph.json b/docs/validation/validation_evidence_graph.json index d1ef52b5..2fe8eb76 100644 --- a/docs/validation/validation_evidence_graph.json +++ b/docs/validation/validation_evidence_graph.json @@ -79,11 +79,13 @@ "scripts/validation_scheduler_experiment.py", "scripts/pytest_scheduler_experiment.py", "scripts/pytest_scheduler_probe.py", + "scripts/pytest_checked_hash_core.py", "scripts/release_check.py", "scripts/validate_release_artifacts.py", ".github/workflows/**", "tests/test_validation_evidence_graph.py", - "tests/test_validation_scheduler_experiment.py" + "tests/test_validation_scheduler_experiment.py", + "tests/test_pytest_checked_hash_core.py" ], "claims": [ "validation-graph-integrity", @@ -185,10 +187,12 @@ "scripts/validation_scheduler_experiment.py", "scripts/pytest_scheduler_experiment.py", "scripts/pytest_scheduler_probe.py", + "scripts/pytest_checked_hash_core.py", "scripts/release_check.py", "scripts/validate_release_artifacts.py", "tests/test_validation_evidence_graph.py", - "tests/test_validation_scheduler_experiment.py" + "tests/test_validation_scheduler_experiment.py", + "tests/test_pytest_checked_hash_core.py" ], "steps": [ { @@ -201,7 +205,8 @@ "-p", "no:cacheprovider", "tests/test_validation_evidence_graph.py", - "tests/test_validation_scheduler_experiment.py" + "tests/test_validation_scheduler_experiment.py", + "tests/test_pytest_checked_hash_core.py" ] } ] diff --git a/docs/validation/validation_lanes.json b/docs/validation/validation_lanes.json index d03c865f..1ac090be 100644 --- a/docs/validation/validation_lanes.json +++ b/docs/validation/validation_lanes.json @@ -41,7 +41,8 @@ "-p", "no:cacheprovider", "tests/test_validation_evidence_graph.py", - "tests/test_validation_scheduler_experiment.py" + "tests/test_validation_scheduler_experiment.py", + "tests/test_pytest_checked_hash_core.py" ] }, { diff --git a/scripts/pytest_checked_hash_core.py b/scripts/pytest_checked_hash_core.py new file mode 100644 index 00000000..e7b47cf7 --- /dev/null +++ b/scripts/pytest_checked_hash_core.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Run one focused producer-core test with a reusable hash-only pycache. + +This is an opt-in local feedback route. The focused test runs once through +``pytest.main`` in this process. Python bytecode writes are disabled for the +test run, while standard checked-hash bytecode is primed afterwards into a +dedicated external prefix on the first invocation. Pytest assertion +rewriting stays in memory, so no rewritten assertion cache is reused. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import sys +import tempfile +from typing import Sequence + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_CACHE_ROOT_NAME = "aoa-session-memory-checked-hash" + +CORE_TARGETS: dict[str, tuple[str, str]] = { + "privacy": ( + "scripts/aoa_session_memory_privacy.py", + "tests/test_session_memory_privacy_core.py", + ), + "outbox": ( + "scripts/aoa_session_memory_outbox.py", + "tests/test_session_memory_outbox_core.py", + ), + "import": ( + "scripts/aoa_session_memory_import.py", + "tests/test_session_memory_import_core.py", + ), +} + + +class CheckedHashError(RuntimeError): + """A cache prefix or initialization input is unsafe or invalid.""" + + +class CachePrimeWarning(CheckedHashError): + """A non-fatal failure while populating the optional warm-cache hint.""" + + +def _external_path(path: Path, label: str) -> Path: + resolved = path.expanduser().resolve() + try: + resolved.relative_to(REPO_ROOT) + except ValueError: + return resolved + raise CheckedHashError(f"{label} must be outside the owner checkout: {resolved}") + + +def _default_cache_prefix() -> Path: + root = Path(os.environ.get("TMPDIR") or tempfile.gettempdir()) + return root / DEFAULT_CACHE_ROOT_NAME + + +def _cache_flags(path: Path) -> int: + try: + with path.open("rb") as handle: + header = handle.read(8) + except OSError as exc: + raise CheckedHashError(f"cannot read bytecode cache {path}: {exc}") from exc + if len(header) < 8: + raise CheckedHashError(f"truncated bytecode cache {path}") + return int.from_bytes(header[4:8], "little") + + +def _validate_prefix(prefix: Path, *, allow_empty: bool) -> bool: + """Validate that *prefix* is empty or contains checked-hash pycs only.""" + if prefix.exists() and not prefix.is_dir(): + raise CheckedHashError(f"cache prefix is not a directory: {prefix}") + if not prefix.exists(): + if not allow_empty: + raise CheckedHashError(f"cache prefix disappeared: {prefix}") + return False + pycs = sorted(prefix.rglob("*.pyc")) + if not pycs: + return False + rewrite = [path for path in pycs if "-pytest-" in path.name] + if rewrite: + raise CheckedHashError( + "cache prefix contains pytest assertion-rewrite bytecode; " + f"choose a fresh dedicated prefix: {rewrite[0]}" + ) + bad = [path for path in pycs if _cache_flags(path) != 3] + if bad: + raise CheckedHashError( + "cache prefix contains non-checked-hash bytecode; " + f"choose a fresh dedicated prefix: {bad[0]}" + ) + return True + + +def _checkout_sources_loaded_before_configuration() -> list[Path]: + """Return owner-checkout sources imported before the cache was configured.""" + loaded: set[Path] = set() + for module in tuple(sys.modules.values()): + filename = getattr(module, "__file__", None) + if not filename or not str(filename).endswith(".py"): + continue + try: + source = Path(filename).resolve() + except (OSError, TypeError, ValueError): + continue + try: + source.relative_to(REPO_ROOT) + except ValueError: + continue + if source != Path(__file__).resolve(): + loaded.add(source) + return sorted(loaded) + + +def _configure_process(prefix: Path) -> None: + """Bind the process to a validated prefix before importing pytest.""" + inherited = sys.pycache_prefix + if inherited is not None: + try: + inherited_path = Path(inherited).expanduser().resolve() + except OSError as exc: + raise CheckedHashError( + f"cannot resolve inherited sys.pycache_prefix {inherited}: {exc}" + ) from exc + if inherited_path != prefix: + raise CheckedHashError( + "inherited sys.pycache_prefix differs from the requested " + f"dedicated prefix ({inherited_path} != {prefix}); unset " + "PYTHONPYCACHEPREFIX or pass the matching --cache-prefix" + ) + preloaded = _checkout_sources_loaded_before_configuration() + if preloaded: + raise CheckedHashError( + "owner-checkout Python sources were imported before cache " + f"configuration: {preloaded[0]}" + ) + if any(name == "pytest" or name.startswith("pytest.") for name in sys.modules): + raise CheckedHashError("pytest was imported before cache configuration") + prefix.mkdir(parents=True, exist_ok=True) + sys.dont_write_bytecode = True + sys.pycache_prefix = str(prefix) + + +def _pytest_args(target: str) -> list[str]: + return [ + "-q", + "-p", + "no:cacheprovider", + "--rootdir", + str(REPO_ROOT), + "--confcutdir", + str(REPO_ROOT), + target, + ] + + +def _module_sources() -> set[Path]: + sources: set[Path] = set() + for module in tuple(sys.modules.values()): + filename = getattr(module, "__file__", None) + if not filename or not str(filename).endswith(".py"): + continue + try: + source = Path(filename).resolve() + except (OSError, TypeError, ValueError): + continue + if source.is_file(): + sources.add(source) + return sources + + +def _prime_checked_hash_bytecode( + *, core_sources: Sequence[Path], selected_test: Path +) -> None: + """Prime imported sources and dynamic product cores without a subprocess.""" + import py_compile + + sources = _module_sources() + explicit = [*core_sources, selected_test] + for source in explicit: + try: + source = source.resolve() + except OSError as exc: + raise CachePrimeWarning( + f"cannot resolve explicit source {source}: {exc}" + ) from exc + if not source.is_file() or source.suffix != ".py": + raise CachePrimeWarning(f"explicit source is unavailable: {source}") + sources.add(source) + + for source in sorted(sources): + try: + py_compile.compile( + str(source), + doraise=True, + invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + ) + except (OSError, ValueError, py_compile.PyCompileError) as exc: + raise CachePrimeWarning( + f"checked-hash bytecode initialization failed for {source}: {exc}" + ) from exc + + try: + ready = _validate_prefix(Path(sys.pycache_prefix), allow_empty=False) + except (CheckedHashError, OSError) as exc: + raise CachePrimeWarning(str(exc)) from exc + if not ready: + raise CachePrimeWarning( + f"checked-hash initialization produced no pycs: {sys.pycache_prefix}" + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Run one focused producer-core pytest route with a reusable " + "checked-hash bytecode prefix and python -B." + ) + ) + parser.add_argument("--core", choices=tuple(CORE_TARGETS), required=True) + parser.add_argument( + "--cache-prefix", + type=Path, + help=( + "external dedicated prefix (default: " + "$TMPDIR/aoa-session-memory-checked-hash)" + ), + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + source, test = CORE_TARGETS[args.core] + try: + prefix = _external_path( + args.cache_prefix or _default_cache_prefix(), "cache prefix" + ) + ready = _validate_prefix(prefix, allow_empty=True) + _configure_process(prefix) + os.chdir(REPO_ROOT) + if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + os.environ["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + + import pytest + + result = pytest.main(_pytest_args(test)) + if not ready: + try: + _prime_checked_hash_bytecode( + core_sources=(REPO_ROOT / source,), + selected_test=REPO_ROOT / test, + ) + except CachePrimeWarning as exc: + print( + f"[checked-hash] warning: cache priming skipped: {exc}", + file=sys.stderr, + ) + else: + print(f"[checked-hash] initialized {prefix}", file=sys.stderr) + return int(result) + except CheckedHashError as exc: + print(f"pytest_checked_hash_core: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_pytest_checked_hash_core.py b/tests/test_pytest_checked_hash_core.py new file mode 100644 index 00000000..eba6d031 --- /dev/null +++ b/tests/test_pytest_checked_hash_core.py @@ -0,0 +1,222 @@ +"""Focused contract tests for the opt-in checked-hash core runner.""" + +from __future__ import annotations + +import importlib.util +import os +from pathlib import Path +import shutil +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "pytest_checked_hash_core_test_source", + ROOT / "scripts" / "pytest_checked_hash_core.py", +) +assert SPEC and SPEC.loader +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + + +def _run_runner( + root: Path, prefix: Path, *, core: str = "import" +) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment.pop("PYTHONPYCACHEPREFIX", None) + environment.pop("PYTHONDONTWRITEBYTECODE", None) + environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + return subprocess.run( + [ + sys.executable, + "-B", + str(root / "scripts" / "pytest_checked_hash_core.py"), + "--core", + core, + "--cache-prefix", + str(prefix), + ], + cwd=root, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + + +def _write_pyc(path: Path, flags: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\x00" * 4 + flags.to_bytes(4, "little") + b"\x00" * 8) + + +def _mutate_preserving_metadata(path: Path, old: bytes, new: bytes) -> None: + assert len(old) == len(new) + before = path.stat() + assert path.read_bytes() == old + path.write_bytes(new) + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + after = path.stat() + assert after.st_size == before.st_size + assert after.st_mtime_ns == before.st_mtime_ns + + +def test_runner_args_keep_assertion_rewrite_and_disable_result_cache() -> None: + arguments = module._pytest_args("tests/test_session_memory_import_core.py") + assert arguments[:3] == ["-q", "-p", "no:cacheprovider"] + assert arguments[-1] == "tests/test_session_memory_import_core.py" + assert "--assert=plain" not in arguments + + +def test_validate_prefix_accepts_checked_hash_only(tmp_path: Path) -> None: + prefix = tmp_path / "hash-only" + _write_pyc(prefix / "module.cpython-314.pyc", 3) + assert module._validate_prefix(prefix, allow_empty=False) is True + + +def test_validate_prefix_rejects_timestamp_or_pytest_rewrite_cache( + tmp_path: Path, +) -> None: + timestamp_prefix = tmp_path / "timestamp" + _write_pyc(timestamp_prefix / "module.cpython-314.pyc", 0) + with pytest.raises(module.CheckedHashError, match="non-checked-hash"): + module._validate_prefix(timestamp_prefix, allow_empty=False) + + rewrite_prefix = tmp_path / "rewrite" + _write_pyc(rewrite_prefix / "test.cpython-314-pytest-9.0.3.pyc", 3) + with pytest.raises(module.CheckedHashError, match="assertion-rewrite"): + module._validate_prefix(rewrite_prefix, allow_empty=False) + + +def test_runner_rejects_unmatched_inherited_prefix(tmp_path: Path) -> None: + requested = tmp_path / "requested" + inherited = tmp_path / "inherited" + environment = dict(os.environ) + environment["PYTHONPYCACHEPREFIX"] = str(inherited) + environment.pop("PYTHONDONTWRITEBYTECODE", None) + result = subprocess.run( + [ + sys.executable, + "-B", + str(ROOT / "scripts" / "pytest_checked_hash_core.py"), + "--core", + "import", + "--cache-prefix", + str(requested), + ], + cwd=ROOT, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + assert result.returncode == 2 + assert "inherited sys.pycache_prefix differs" in result.stdout + + +def test_runner_reports_checkout_prefix_as_normal_cli_error() -> None: + result = _run_runner(ROOT, ROOT / ".pytest-checked-hash-inside") + assert result.returncode == 2 + assert "must be outside the owner checkout" in result.stdout + + +def test_runner_catches_same_size_same_mtime_product_test_and_auxiliary_mutations( + tmp_path: Path, +) -> None: + root = tmp_path / "checkout" + scripts = root / "scripts" + tests = root / "tests" + scripts.mkdir(parents=True) + tests.mkdir() + shutil.copy2( + ROOT / "scripts" / "pytest_checked_hash_core.py", + scripts / "pytest_checked_hash_core.py", + ) + for core in ("privacy", "outbox"): + (scripts / f"aoa_session_memory_{core}.py").write_text( + 'VALUE = "unused"\n', encoding="utf-8" + ) + product = scripts / "aoa_session_memory_import.py" + auxiliary = scripts / "auxiliary.py" + test = tests / "test_session_memory_import_core.py" + product_old = ( + b'from scripts.auxiliary import VALUE as AUXILIARY_VALUE\nVALUE = "old"\n' + ) + product_new = ( + b'from scripts.auxiliary import VALUE as AUXILIARY_VALUE\nVALUE = "new"\n' + ) + auxiliary_old = b'VALUE = "old"\n' + auxiliary_new = b'VALUE = "new"\n' + test_old = ( + b"from scripts.aoa_session_memory_import import AUXILIARY_VALUE, VALUE\n" + b"\n" + b"def test_values_are_old():\n" + b' assert VALUE == "old"\n' + b' assert AUXILIARY_VALUE == "old"\n' + ) + test_new = test_old.replace(b"test_values_are_old", b"test_values_are_new").replace( + b'VALUE == "old"', b'VALUE == "new"', 1 + ) + for path, content in ( + (product, product_old), + (auxiliary, auxiliary_old), + (test, test_old), + ): + path.write_bytes(content) + + prefix = tmp_path / "cache" + initial = _run_runner(root, prefix) + assert initial.returncode == 0, initial.stdout + assert list(prefix.rglob("*.pyc")) + assert not list(prefix.rglob("*-pytest-*.pyc")) + + for path, old, new in ( + (product, product_old, product_new), + (auxiliary, auxiliary_old, auxiliary_new), + (test, test_old, test_new), + ): + before = path.stat() + _mutate_preserving_metadata(path, old, new) + try: + changed = _run_runner(root, prefix) + assert changed.returncode == 1, (path, changed.stdout) + finally: + path.write_bytes(old) + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + restored = _run_runner(root, prefix) + assert restored.returncode == 0, (path, restored.stdout) + + +def test_runner_preserves_pytest_status_when_cache_priming_fails( + tmp_path: Path, +) -> None: + root = tmp_path / "checkout" + scripts = root / "scripts" + tests = root / "tests" + scripts.mkdir(parents=True) + tests.mkdir() + shutil.copy2( + ROOT / "scripts" / "pytest_checked_hash_core.py", + scripts / "pytest_checked_hash_core.py", + ) + product = scripts / "aoa_session_memory_import.py" + product.write_text('VALUE = "ok"\n', encoding="utf-8") + test = tests / "test_session_memory_import_core.py" + test.write_text( + "from pathlib import Path\n" + "from scripts.aoa_session_memory_import import VALUE\n" + "\n" + "def test_ok_then_remove_source():\n" + ' assert VALUE == "ok"\n' + ' Path(__file__).parents[1].joinpath("scripts", ' + '"aoa_session_memory_import.py").unlink()\n', + encoding="utf-8", + ) + + result = _run_runner(root, tmp_path / "cache") + assert result.returncode == 0, result.stdout + assert "cache priming skipped" in result.stdout