From 9fee9167a15f012295c6954239ff63f3cb7a69f4 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 01:49:42 +0000 Subject: [PATCH 01/23] Make v0.2 distribution provenance and artifact builds reproducible --- MANIFEST.in | 7 ++ README.md | 28 +++++++- pyproject.toml | 3 + requirements-build.txt | 5 ++ src/pubskill_lib/_source.json | 6 ++ src/pubskill_lib/audit.py | 8 +-- tests/test_provenance.py | 10 ++- tools/build_release.py | 119 ++++++++++++++++++++++++++++++++++ 8 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 MANIFEST.in create mode 100644 requirements-build.txt create mode 100644 src/pubskill_lib/_source.json create mode 100644 tools/build_release.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..18d92c9 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include AGENTS.md SOURCE.md HANDOFF.md HANDOFF.vm.md .env.example +include requirements-build.txt +graft .agents/skills +graft examples +graft tests +graft tools +global-exclude __pycache__ *.py[cod] diff --git a/README.md b/README.md index 84e78d2..4088b43 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,14 @@ Clone this repo when you want a command that inspects a local repository and wri ## Status -The inspect CLI implementation passes the repository gate; the `v0.2.0` release tag is not published yet. +The inspect CLI version is `0.2.0`. Published versions and their immutable +artifacts are listed on [GitHub Releases](https://github.com/The-Interdependency/pubskill-lib/releases). | Claim | State | |---|---| | Canon | `The-Interdependency/skill-lib` | | This repo | distribution + public CLI + fixtures | -| Clone / run / findings | **implementation ready** — release pending | +| Clone / run / findings | Source and built-artifact gates described below | | VM populate | `HANDOFF.vm.md` | | Source pin | `SOURCE.md` | @@ -31,6 +32,29 @@ python -m pubskill_lib.audit examples/neglected-repo --out /tmp/findings.json Those commands are the definition of done for the first utility tag (`v0.2.0`). They run in GitHub CI from a clean checkout; publish the tag only after the release gate is explicitly completed. +## Reproduce release artifacts + +From the release's exact Git commit, install the pinned build tools and build +into two empty directories: + +```bash +python -m pip install -r requirements-build.txt +python tools/build_release.py --out /tmp/pubskill-build-a +python tools/build_release.py --out /tmp/pubskill-build-b +diff /tmp/pubskill-build-a/SHA256SUMS /tmp/pubskill-build-b/SHA256SUMS +``` + +The builder uses only committed source, normalizes source archive headers, and +records source, doctrine, toolchain, and artifact digests in `release-manifest.json`. +It does not publish. Before publication, install the exact wheel in a fresh venv, +run the tests and fixture from the extracted sdist, and inspect a real consumer. +The wheel retains its canonical skill-lib source pin without requiring a checkout. + +Download the wheel, source archive, manifest, and `SHA256SUMS` from the chosen +release. Verify the downloaded files with `sha256sum -c SHA256SUMS`, then install +the verified wheel with `python -m pip install --no-deps ./pubskill_lib-0.2.0-py3-none-any.whl`. +Checksums establish byte identity; they are not a signature or a blanket health claim. + ## Inspect CLI `v0.2` inspects one **local repository path** without executing the target repository: diff --git a/pyproject.toml b/pyproject.toml index ecfd183..4a68b1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,3 +16,6 @@ pubskill-examine = "pubskill_lib.examine:main" [tool.setuptools.packages.find] where = ["src"] + +[tool.setuptools.package-data] +pubskill_lib = ["_source.json"] diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 0000000..67e6895 --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1,5 @@ +build==1.6.1 +packaging==26.3 +pyproject-hooks==1.2.0 +setuptools==84.0.0 +wheel==0.48.0 diff --git a/src/pubskill_lib/_source.json b/src/pubskill_lib/_source.json new file mode 100644 index 0000000..a4e6110 --- /dev/null +++ b/src/pubskill_lib/_source.json @@ -0,0 +1,6 @@ +{ + "schema": "pubskill-lib.source", + "version": 1, + "repository": "The-Interdependency/skill-lib", + "commit": "8de4f12d0f31ff94f41e4a0196c447c0cbe20faf" +} diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 90b09ea..7f2efb7 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -9,6 +9,7 @@ import argparse import json +from importlib.resources import files import re import shlex import subprocess @@ -277,10 +278,9 @@ def _check_package_scripts(target, sink): def _read_source_pin(): - root = Path(__file__).resolve().parents[2] - text = _read_text(root / "SOURCE.md") or "" - match = PIN_PATTERN.search(text) - return match.group(1) if match else "hmmm" + """Read the canonical identity shipped with both source and wheel installs.""" + data = json.loads(files("pubskill_lib").joinpath("_source.json").read_text(encoding="utf-8")) + return data["commit"] def audit_path(target_path, source_pin=None): diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 79202c7..3172e2d 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -17,6 +17,10 @@ def _source_pin() -> str: class PublicationProvenanceTests(unittest.TestCase): + def test_installed_source_pin_matches_publication_pin(self): + from pubskill_lib.audit import _read_source_pin + self.assertEqual(_source_pin(), _read_source_pin()) + def test_source_pin_matches_vendored_skill_manifest(self): vendored = (REPO / ".agents" / "skills" / "README.md").read_text(encoding="utf-8") vendored_pin = PIN_RE.search(vendored) @@ -42,10 +46,10 @@ def test_local_secret_files_are_ignored(self): self.assertIn(".env.*", ignore) self.assertIn("*.egg-info/", ignore) - def test_readme_does_not_claim_unpublished_v020_tag(self): + def test_readme_exposes_reproducible_release_gate(self): readme = (REPO / "README.md").read_text(encoding="utf-8") - self.assertIn("release pending", readme) - self.assertNotIn("**shipped** — `v0.2`", readme) + self.assertIn("tools/build_release.py", readme) + self.assertIn("sha256sum -c SHA256SUMS", readme) if __name__ == "__main__": diff --git a/tools/build_release.py b/tools/build_release.py new file mode 100644 index 0000000..0161f33 --- /dev/null +++ b/tools/build_release.py @@ -0,0 +1,119 @@ +# === MODULE_BUILD === +# id: pubskill_release_builder +# module_name: build_release +# module_kind: instrument +# summary: builds normalized immutable wheel and sdist artifacts from a clean exact Git commit +# owner: The Interdependency +# public_surface: python tools/build_release.py --out DIRECTORY +# internal_surface: normalize_sdist, main +# auth_boundary: none +# storage_boundary: temporary build directory and explicit output directory +# network_boundary: none; build dependencies must already be installed +# user_data_boundary: none +# admin_only: false +# tests: clean-install repository suite and two-build digest comparison documented in README +# rollout: explicit release build command +# rollback: return to previous published immutable release +# === END MODULE_BUILD === +# === CONTRACTS === +# id: release_build_binds_exact_source +# given: a clean source checkout and the pinned build toolchain +# then: artifacts derive only from Git HEAD; the manifest records source, doctrine, toolchain and output digests +# class: provenance +# === END CONTRACTS === + +"""Usage: install requirements-build.txt, then run with --out /tmp/release. + +Run twice into separate empty directories and compare wheel/sdist SHA-256 values. +The builder performs no publication. Clean-install and consumer gates are required +before publishing these bytes. Tar headers are normalized to the commit timestamp; +wheel timestamps use SOURCE_DATE_EPOCH. Source file contents are unchanged. +""" +from __future__ import annotations + +import argparse +import gzip +import hashlib +import importlib.metadata +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile + + +def normalize_sdist(path: Path, destination: Path, epoch: int) -> None: + with path.open("rb") as raw, tarfile.open(fileobj=raw, mode="r:gz") as source: + with destination.open("wb") as output, gzip.GzipFile(filename="", mode="wb", fileobj=output, mtime=epoch) as compressed: + with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as target: + for member in sorted(source.getmembers(), key=lambda item: item.name): + if not (member.isfile() or member.isdir()): + raise ValueError(f"unexpected sdist member: {member.name}") + member.uid = member.gid = 0 + member.uname = member.gname = "" + member.mtime = epoch + member.pax_headers = {} + member.mode = 0o755 if member.isdir() or member.mode & 0o111 else 0o644 + if member.isfile(): + with source.extractfile(member) as stream: + target.addfile(member, stream) + else: + target.addfile(member) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + def git(*args): + return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() + if git("status", "--porcelain"): + raise SystemExit("release build requires a clean Git checkout") + commit = git("rev-parse", "HEAD") + epoch = int(git("show", "-s", "--format=%ct", commit)) + out = args.out.resolve() + out.mkdir(parents=True, exist_ok=True) + if any(out.iterdir()): + raise SystemExit("release output directory must be empty") + versions = {} + for requirement in (root / "requirements-build.txt").read_text().splitlines(): + name, version = requirement.split("==") + versions[name] = importlib.metadata.version(name) + if versions[name] != version: + raise SystemExit(f"build toolchain mismatch: {name}") + with tempfile.TemporaryDirectory(prefix="pubskill-release-") as directory: + temporary = Path(directory) + source = temporary / "source" + source.mkdir() + archive = subprocess.check_output(["git", "-C", str(root), "archive", commit]) + with tarfile.open(fileobj=io.BytesIO(archive)) as tree: + for member in tree.getmembers(): + if member.name.startswith("/") or ".." in Path(member.name).parts or not (member.isfile() or member.isdir()): + raise ValueError("unsafe source archive") + tree.extractall(source) + environment = dict(os.environ, SOURCE_DATE_EPOCH=str(epoch), PYTHONHASHSEED="0") + environment.pop("PYTHONPATH", None) + subprocess.run([sys.executable, "-m", "build", "--no-isolation", "--outdir", str(temporary / "dist"), str(source)], check=True, env=environment) + for artifact in sorted((temporary / "dist").iterdir()): + if artifact.name.endswith(".tar.gz"): + normalize_sdist(artifact, out / artifact.name, epoch) + elif artifact.suffix == ".whl": + (out / artifact.name).write_bytes(artifact.read_bytes()) + else: + raise ValueError(f"unexpected build artifact: {artifact.name}") + hashes = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(out.iterdir())} + doctrine = json.loads((root / "src/pubskill_lib/_source.json").read_text()) + manifest = {"schema": "pubskill-lib.release-manifest", "version": 1, "source_commit": commit, "source_tree": git("rev-parse", "HEAD^{tree}"), "source_date_epoch": epoch, "skill_lib_commit": doctrine["commit"], "build_toolchain": versions, "artifacts_sha256": hashes} + receipt = out / "release-manifest.json" + receipt.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + hashes[receipt.name] = hashlib.sha256(receipt.read_bytes()).hexdigest() + (out / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in sorted(hashes.items()))) + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() From fcb6a7b856cf94a201710b6044200500ce19d5c5 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 01:52:08 +0000 Subject: [PATCH 02/23] Replay complete source fixtures against reproducible wheel artifacts --- .github/workflows/ci.yml | 14 ++++++++++++++ MANIFEST.in | 1 + tools/build_release.py | 1 + 3 files changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ad6459..d7aae68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,3 +27,17 @@ jobs: /tmp/pubskill-wheel-venv/bin/python -m pip install --no-deps /tmp/pubskill-wheel/pubskill_lib-*.whl cd /tmp /tmp/pubskill-wheel-venv/bin/python -c "from pubskill_lib import evidence; assert evidence._comment_markers()['.py'] == '#'" + - name: Reproduce release artifacts and replay installed wheel + run: | + python -m venv /tmp/pubskill-build-venv + /tmp/pubskill-build-venv/bin/python -m pip install -r requirements-build.txt + /tmp/pubskill-build-venv/bin/python tools/build_release.py --out /tmp/pubskill-release-a + /tmp/pubskill-build-venv/bin/python tools/build_release.py --out /tmp/pubskill-release-b + diff /tmp/pubskill-release-a/SHA256SUMS /tmp/pubskill-release-b/SHA256SUMS + /tmp/pubskill-wheel-venv/bin/python -m pip install --no-deps --force-reinstall /tmp/pubskill-release-a/*.whl + mkdir /tmp/pubskill-replay + tar -xzf /tmp/pubskill-release-a/*.tar.gz -C /tmp/pubskill-replay + cd /tmp/pubskill-replay/pubskill_lib-0.2.0 + /tmp/pubskill-wheel-venv/bin/python -m unittest discover -s tests + /tmp/pubskill-wheel-venv/bin/python -m pubskill_lib.audit examples/neglected-repo --out /tmp/release-findings.json + /tmp/pubskill-wheel-venv/bin/python -c "import json; from pubskill_lib.audit import _read_source_pin; assert json.load(open('/tmp/release-findings.json'))['source_pin'] == _read_source_pin() != 'hmmm'" diff --git a/MANIFEST.in b/MANIFEST.in index 18d92c9..94e107f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include AGENTS.md SOURCE.md HANDOFF.md HANDOFF.vm.md .env.example +include .gitignore include requirements-build.txt graft .agents/skills graft examples diff --git a/tools/build_release.py b/tools/build_release.py index 0161f33..52c40e2 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -110,6 +110,7 @@ def git(*args): manifest = {"schema": "pubskill-lib.release-manifest", "version": 1, "source_commit": commit, "source_tree": git("rev-parse", "HEAD^{tree}"), "source_date_epoch": epoch, "skill_lib_commit": doctrine["commit"], "build_toolchain": versions, "artifacts_sha256": hashes} receipt = out / "release-manifest.json" receipt.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + hashes = dict(hashes) hashes[receipt.name] = hashlib.sha256(receipt.read_bytes()).hexdigest() (out / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in sorted(hashes.items()))) print(json.dumps(manifest, indent=2)) From 66c299c4daf3b43dcfecdb9cfc24139c515a97d2 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 01:57:29 +0000 Subject: [PATCH 03/23] Preserve BOMs across adapters and complete Node option arity --- src/pubskill_lib/audit.py | 33 ++++++++++++++++++++++----------- src/pubskill_lib/evidence.py | 3 ++- tests/test_repairs.py | 18 +++++++++++++++++- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 7f2efb7..1693675 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -30,7 +30,7 @@ PIN_PATTERN = re.compile(r"`([0-9a-f]{40})`") LOCAL_SCRIPT_INTERPRETERS = {"node", "python", "python3", "bash", "sh"} NON_FILE_MODES = { - "node": {"-e", "--eval", "-p", "--print"}, + "node": {"-e", "--eval", "-p", "--print", "--run"}, "python": {"-c", "-m"}, "python3": {"-c", "-m"}, "bash": {"-c"}, @@ -42,16 +42,27 @@ "bash": {"-o", "+o", "-O", "+O", "--rcfile", "--init-file"}, "sh": {"-o", "+o"}, "node": { - "-r", "--require", "--import", "--loader", "--experimental-loader", - "--conditions", "-C", "--input-type", "--env-file", - "--env-file-if-exists", "--inspect-port", "--inspect-publish-uid", - "--title", "--icu-data-dir", "--openssl-config", "--redirect-warnings", - "--trace-event-categories", "--trace-event-file-pattern", - "--unhandled-rejections", "--diagnostic-dir", "--report-directory", - "--report-filename", "--test-reporter", "--test-reporter-destination", - "--test-name-pattern", "--test-skip-pattern", "--test-concurrency", - "--test-shard", "--test-timeout", "--max-old-space-size", - "--stack-trace-limit", + "--allow-fs-read", "--allow-fs-write", "--build-snapshot-config", "--conditions", + "--cpu-prof-dir", "--cpu-prof-interval", "--cpu-prof-name", "--debug-port", + "--diagnostic-dir", "--disable-proto", "--disable-warning", "--dns-result-order", + "--env-file", "--env-file-if-exists", "--experimental-config-file", + "--experimental-default-type", "--experimental-loader", "--experimental-sea-config", + "--experimental-test-isolation", "--heap-prof-dir", "--heap-prof-interval", + "--heap-prof-name", "--heapsnapshot-near-heap-limit", "--heapsnapshot-signal", + "--icu-data-dir", "--import", "--input-type", "--inspect-port", + "--inspect-publish-uid", "--loader", "--localstorage-file", "--max-http-header-size", + "--max-old-space-size", "--max-old-space-size-percentage", + "--network-family-autoselection-attempt-timeout", "--openssl-config", + "--redirect-warnings", "--report-dir", "--report-directory", "--report-filename", + "--report-signal", "--require", "--secure-heap", "--secure-heap-min", + "--snapshot-blob", "--stack-trace-limit", "--test-concurrency", + "--test-coverage-branches", "--test-coverage-exclude", "--test-coverage-functions", + "--test-coverage-include", "--test-coverage-lines", "--test-name-pattern", + "--test-reporter", "--test-reporter-destination", "--test-shard", + "--test-skip-pattern", "--test-timeout", "--title", "--tls-cipher-list", + "--tls-keylog", "--trace-event-categories", "--trace-event-file-pattern", + "--trace-require-module", "--unhandled-rejections", "--use-largepages", + "--v8-pool-size", "--watch-kill-signal", "--watch-path", "-C", "-r", }, } diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index bb65bc5..5bb6e09 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -101,7 +101,8 @@ def _decode_source(path: Path, raw: bytes) -> tuple[str | None, str | None, str except (LookupError, SyntaxError, UnicodeDecodeError) as exc: return None, None, f"source encoding unresolved: {exc}" try: - return raw.decode("utf-8"), "utf-8", None + encoding = "utf-8-sig" if raw.startswith(b"\xef\xbb\xbf") else "utf-8" + return raw.decode(encoding), encoding, None except UnicodeDecodeError as exc: return None, None, f"source encoding unresolved: {exc}" diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 5a86f06..9cf3726 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -161,6 +161,21 @@ def chat(self, system, user): self.assertEqual([], report["changed"]) self.assertIn("tool.py", report["hmmm"]) + def test_non_python_bom_stays_at_byte_zero(self): + for name, body in (("main.c", "int main(void) { return 0; }\n"), ("main.rs", "fn main() {}\n")): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / name + path.write_bytes(body.encode("utf-8-sig")) + before = evidence.read_evidence(root, path) + examine._apply(root, [before], [], False) + raw = path.read_bytes() + self.assertTrue(raw.startswith(b"\xef\xbb\xbf")) + self.assertEqual(1, raw.count(b"\xef\xbb\xbf")) + self.assertEqual(before.sha256, evidence.read_evidence(root, path).sha256) + examine._apply(root, [evidence.read_evidence(root, path)], [], False) + self.assertEqual(raw, path.read_bytes()) + def test_adapterless_narratives_are_retained_without_writes(self): for filename, marker in (("tool.sh", "#"), ("index.php", "//")): with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp: @@ -280,6 +295,7 @@ def test_value_taking_interpreter_options_select_actual_file(self): "node --require preload.js app.py", "node -rpreload.js app.py", "node --import preload.js --trace-warnings app.py", "node --max-old-space-size 512 app.py", + "node --watch --watch-path src app.py", "bash -o errexit app.py", "bash -O extglob app.py", "bash --rcfile startup.sh app.py", "bash -eo pipefail app.py", "sh +o errexit app.py", "python -- app.py", @@ -301,4 +317,4 @@ def test_non_object_package_manifest_is_target_defect(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 60233574cfc21179200112ce4948ee4842709d21 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 02:13:22 +0000 Subject: [PATCH 04/23] Handle Node test runner value options and aliases --- src/pubskill_lib/audit.py | 5 +++-- tests/test_repairs.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 1693675..d0196bb 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -47,17 +47,18 @@ "--diagnostic-dir", "--disable-proto", "--disable-warning", "--dns-result-order", "--env-file", "--env-file-if-exists", "--experimental-config-file", "--experimental-default-type", "--experimental-loader", "--experimental-sea-config", - "--experimental-test-isolation", "--heap-prof-dir", "--heap-prof-interval", + "--experimental-package-map", "--experimental-test-tag-filter", "--experimental-test-isolation", "--heap-prof-dir", "--heap-prof-interval", "--heap-prof-name", "--heapsnapshot-near-heap-limit", "--heapsnapshot-signal", "--icu-data-dir", "--import", "--input-type", "--inspect-port", "--inspect-publish-uid", "--loader", "--localstorage-file", "--max-http-header-size", - "--max-old-space-size", "--max-old-space-size-percentage", + "--max-old-space-size", "--max-old-space-size-percentage", "--max-semi-space-size", "--network-family-autoselection-attempt-timeout", "--openssl-config", "--redirect-warnings", "--report-dir", "--report-directory", "--report-filename", "--report-signal", "--require", "--secure-heap", "--secure-heap-min", "--snapshot-blob", "--stack-trace-limit", "--test-concurrency", "--test-coverage-branches", "--test-coverage-exclude", "--test-coverage-functions", "--test-coverage-include", "--test-coverage-lines", "--test-name-pattern", + "--test-global-setup", "--test-isolation", "--test-random-seed", "--test-rerun-failures", "--test-reporter", "--test-reporter-destination", "--test-shard", "--test-skip-pattern", "--test-timeout", "--title", "--tls-cipher-list", "--tls-keylog", "--trace-event-categories", "--trace-event-file-pattern", diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 9cf3726..cd359bf 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -296,6 +296,12 @@ def test_value_taking_interpreter_options_select_actual_file(self): "node --import preload.js --trace-warnings app.py", "node --max-old-space-size 512 app.py", "node --watch --watch-path src app.py", + "node --test-isolation none app.py", + "node --experimental-test-isolation none app.py", + "node --test-global-setup setup.js app.py", + "node --test-rerun-failures failures.json app.py", + "node --test-random-seed 12 app.py", + "node --max-semi-space-size 16 app.py", "bash -o errexit app.py", "bash -O extglob app.py", "bash --rcfile startup.sh app.py", "bash -eo pipefail app.py", "sh +o errexit app.py", "python -- app.py", From a07c16d2f039817419723d8ddd0fe86bc49ab11b Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 02:31:01 +0000 Subject: [PATCH 05/23] Pin isolated backend and replay the clean source installation --- .github/workflows/ci.yml | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7aae68..395c820 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,3 +41,7 @@ jobs: /tmp/pubskill-wheel-venv/bin/python -m unittest discover -s tests /tmp/pubskill-wheel-venv/bin/python -m pubskill_lib.audit examples/neglected-repo --out /tmp/release-findings.json /tmp/pubskill-wheel-venv/bin/python -c "import json; from pubskill_lib.audit import _read_source_pin; assert json.load(open('/tmp/release-findings.json'))['source_pin'] == _read_source_pin() != 'hmmm'" + python -m venv /tmp/pubskill-sdist-venv + /tmp/pubskill-sdist-venv/bin/python -m pip install --no-deps /tmp/pubskill-release-a/*.tar.gz + /tmp/pubskill-sdist-venv/bin/python -m unittest discover -s tests + /tmp/pubskill-sdist-venv/bin/python -c "from pubskill_lib.audit import _read_source_pin; assert _read_source_pin() != 'hmmm'" diff --git a/pyproject.toml b/pyproject.toml index 4a68b1e..ed03fd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68"] +requires = ["setuptools==84.0.0", "wheel==0.48.0"] build-backend = "setuptools.build_meta" [project] From 57c46a2925151d7348fa635abd08ca414d4881c3 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 02:35:20 +0000 Subject: [PATCH 06/23] Preserve shell quoting, URL entrypoints, and canonical parser bytes --- src/pubskill_lib/audit.py | 49 ++++++++++++++++++++++++++++++++++--- src/pubskill_lib/examine.py | 13 ++++++++-- tests/test_repairs.py | 40 ++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index d0196bb..5fc0a25 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -12,6 +12,7 @@ from importlib.resources import files import re import shlex +from urllib.parse import unquote, urlsplit import subprocess import sys from pathlib import Path @@ -198,6 +199,39 @@ def _check_pyproject_scripts(target, sink): ) +def _shell_segments(command): + """Split direct shell commands while retaining quoted/escaped separators.""" + start, quote, escaped = 0, None, False + for index, character in enumerate(command): + if escaped: + escaped = False + elif character == "\\" and quote != "'": + escaped = True + elif quote: + if character == quote: + quote = None + elif character in {"'", '"'}: + quote = character + elif character in ";&|\n": + yield command[start:index] + start = index + 1 + yield command[start:] + + +def _entrypoint_target(token, entry_url): + if not entry_url: + return token + try: + parsed = urlsplit(token) + if parsed.scheme == "file" and parsed.netloc in {"", "localhost"}: + return unquote(parsed.path, errors="strict") + if not parsed.scheme: + return unquote(parsed.path, errors="strict") # Relative entry URL. + except (ValueError, UnicodeError): + pass + return None + + def _local_script_targets(command): """Yield direct file operands after documented interpreter options. @@ -205,7 +239,7 @@ def _local_script_targets(command): Python -W/-X, Bash -o/-O and startup files, and common Node value options consume their arguments; attached values and -- delimiters are supported. """ - for segment in re.split(r"\s*(?:&&|;|\|)\s*", command): + for segment in _shell_segments(command): if not segment.strip(): continue try: @@ -216,12 +250,19 @@ def _local_script_targets(command): continue interpreter = tokens[0] non_file_modes = NON_FILE_MODES[interpreter] + entry_url = False index = 1 while index < len(tokens): token = tokens[index] + if interpreter == "node" and token in {"--entry-url", "--experimental-entry-url"}: + entry_url = True + index += 1 + continue if token == "--": if index + 1 < len(tokens) and tokens[index + 1] != "-": - yield tokens[index + 1] + target = _entrypoint_target(tokens[index + 1], entry_url) + if target is not None: + yield target break if token == "-" or token.split("=", 1)[0] in non_file_modes: break @@ -247,7 +288,9 @@ def _local_script_targets(command): break index += 1 continue - yield token + target = _entrypoint_target(token, entry_url) + if target is not None: + yield target break diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index f782671..ac3ff21 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -26,12 +26,17 @@ from . import ratios +def _canonical_artifact(path: Path) -> bool: + return path.name == "_msdmd_universal.py" and path.parent.name == "pubskill_lib" + + def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: - supported = [ev for ev in evidence_list if ev.marker is not None] + supported = [ev for ev in evidence_list if ev.marker is not None and not _canonical_artifact(Path(ev.path))] return { "root": str(root), "files": len(evidence_list), "supported_files": len(supported), + "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(Path(ev.path))], "unsupported": [ {"path": ev.path, "hmmm": ev.hmmm} for ev in evidence_list if ev.marker is None ], @@ -49,6 +54,7 @@ def _apply( narratives: dict[str, dict[str, str]] = {} changed: list[str] = [] unresolved: dict[str, str] = {} + preserved_authority: list[str] = [] now = datetime.now(timezone.utc).isoformat() engine = ratios.RatiosEngine() @@ -56,6 +62,9 @@ def _apply( path = boundary.assert_inside(root, root / ev.path) if ev.narrative_entries: narratives[ev.path] = ev.narrative_entries[0] + if _canonical_artifact(path): + preserved_authority.append(ev.path) + continue adapter = engine.adapter_for(path) if ev.marker is None or adapter is None or ev.encoding is None: continue @@ -104,7 +113,7 @@ def _apply( narratives[ev.path] = entry changed.extend(file_changes) - return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved} + return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved, "preserved_authority": preserved_authority} def main(argv: list[str] | None = None) -> int: diff --git a/tests/test_repairs.py b/tests/test_repairs.py index cd359bf..04f7464 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -47,6 +47,20 @@ def test_blank_model_override_uses_provider_default(self): class NarrativeBoundaryTests(unittest.TestCase): + def test_apply_preserves_packaged_canonical_parser(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + parser = root / "src/pubskill_lib/_msdmd_universal.py" + parser.parent.mkdir(parents=True) + original = Path(evidence._canonical_msdmd.__file__).read_bytes() + parser.write_bytes(original) + ev = evidence.read_evidence(root, parser) + _, report = examine._apply(root, [ev], [], True) + self.assertEqual(original, parser.read_bytes()) + self.assertEqual([], report["changed"]) + self.assertEqual([ev.path], report["preserved_authority"]) + self.assertEqual(0, examine._plan(root, [ev])["supported_files"]) + def test_narrative_preserves_python_shebang_and_encoding_header(self): text = ( "#!/usr/bin/env python3\n" @@ -302,6 +316,11 @@ def test_value_taking_interpreter_options_select_actual_file(self): "node --test-rerun-failures failures.json app.py", "node --test-random-seed 12 app.py", "node --max-semi-space-size 16 app.py", + "node --test-name-pattern 'unit|integration' app.py", + "node --test-name-pattern 'unit;integration' app.py", + "node --test-name-pattern '|' app.py", + "node --test-name-pattern unit\\|integration app.py", + "node --inspect=9229 app.py", "node --inspect app.py", "bash -o errexit app.py", "bash -O extglob app.py", "bash --rcfile startup.sh app.py", "bash -eo pipefail app.py", "sh +o errexit app.py", "python -- app.py", @@ -313,6 +332,27 @@ def test_value_taking_interpreter_options_select_actual_file(self): with self.subTest(command=command): self.assertEqual([], list(audit._local_script_targets(command))) + def test_inspector_endpoint_requires_equals_in_node_24(self): + # Official Node v24.15.0 attempts to load 9229 as the entry file here. + self.assertEqual(["9229"], list(audit._local_script_targets("node --inspect 9229 app.js"))) + + def test_quoted_segments_and_entrypoint_urls(self): + self.assertEqual(["first.js", "second.js"], list(audit._local_script_targets("node --test-name-pattern 'a|b' first.js && node second.js"))) + self.assertEqual([], list(audit._local_script_targets("node --entry-url 'data:text/javascript,console.log(1);'"))) + self.assertEqual(["/definitely/missing file.js"], list(audit._local_script_targets("node --entry-url file:///definitely/missing%20file.js"))) + self.assertEqual(["./local file.js"], list(audit._local_script_targets("node --entry-url './local%20file.js?debug=1#part'"))) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "package.json").write_text(json.dumps({"scripts": { + "file": "node --entry-url file:///definitely/missing.js", + "data": "node --entry-url 'data:text/javascript,console.log(1);'", + "quoted": "node --test-name-pattern 'unit|integration' missing.js", + }})) + claims = [f["claim"] for f in audit.audit_path(root, "pin")["findings"] if f["surface"] == "deps"] + self.assertEqual(2, len(claims)) + self.assertTrue(any("escapes repository via /definitely/missing.js" in claim for claim in claims)) + self.assertTrue(any("missing local file missing.js" in claim for claim in claims)) + def test_non_object_package_manifest_is_target_defect(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From ee60fdbb69660dee438ed583836943d152ab7ec9 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 03:01:05 +0000 Subject: [PATCH 07/23] Preserve literal source data and concurrent edits during examination --- src/pubskill_lib/audit.py | 60 ++++++++++++++------- src/pubskill_lib/evidence.py | 35 +++++------- src/pubskill_lib/examine.py | 15 ++++-- src/pubskill_lib/msdmd_writer.py | 48 +++++++++-------- src/pubskill_lib/ratios.py | 19 +++---- src/pubskill_lib/source_boundaries.py | 40 ++++++++++++++ tests/test_repairs.py | 78 +++++++++++++++++++++++++++ 7 files changed, 218 insertions(+), 77 deletions(-) create mode 100644 src/pubskill_lib/source_boundaries.py diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 5fc0a25..ab414e5 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -218,21 +218,26 @@ def _shell_segments(command): yield command[start:] -def _entrypoint_target(token, entry_url): - if not entry_url: - return token +def _entrypoint_target(token, entry_url, unresolved=None): try: - parsed = urlsplit(token) - if parsed.scheme == "file" and parsed.netloc in {"", "localhost"}: - return unquote(parsed.path, errors="strict") - if not parsed.scheme: - return unquote(parsed.path, errors="strict") # Relative entry URL. - except (ValueError, UnicodeError): - pass + target = token + if entry_url: + parsed = urlsplit(token) + if parsed.scheme == "file" and parsed.netloc not in {"", "localhost"}: + raise ValueError("unsupported file URL authority") + if parsed.scheme not in {"", "file"}: + return None + target = unquote(parsed.path, errors="strict") + if not target or "\0" in target: + raise ValueError("empty or NUL-containing path") + return target + except (ValueError, UnicodeError) as error: + if unresolved is not None: + unresolved.append(f"unresolved entrypoint {token!r}: {error}") return None -def _local_script_targets(command): +def _local_script_targets(command, unresolved=None): """Yield direct file operands after documented interpreter options. This is a static audit of direct invocations, not a shell evaluator. @@ -254,13 +259,26 @@ def _local_script_targets(command): index = 1 while index < len(tokens): token = tokens[index] + if interpreter == "node" and token == "inspect": + arguments = tokens[index + 1:] + if not arguments: + break + target = arguments[0] + if re.fullmatch(r"[^:]+:\d+", target) or (len(arguments) == 2 and target == "-p" and arguments[1].isdigit()): + break # Attach to an existing debugger/process, not a file. + if re.fullmatch(r"--port=\d+", target): + target = arguments[1] if len(arguments) > 1 else "" + target = _entrypoint_target(target, False, unresolved) + if target is not None: + yield target + break if interpreter == "node" and token in {"--entry-url", "--experimental-entry-url"}: entry_url = True index += 1 continue if token == "--": if index + 1 < len(tokens) and tokens[index + 1] != "-": - target = _entrypoint_target(tokens[index + 1], entry_url) + target = _entrypoint_target(tokens[index + 1], entry_url, unresolved) if target is not None: yield target break @@ -288,13 +306,13 @@ def _local_script_targets(command): break index += 1 continue - target = _entrypoint_target(token, entry_url) + target = _entrypoint_target(token, entry_url, unresolved) if target is not None: yield target break -def _check_package_scripts(target, sink): +def _check_package_scripts(target, sink, unresolved): package = target / "package.json" text = _read_text(package) if text is None: @@ -313,12 +331,17 @@ def _check_package_scripts(target, sink): for name, command in sorted(scripts.items()): if not isinstance(command, str): continue - for raw_path in _local_script_targets(command): + script_unresolved = [] + for raw_path in _local_script_targets(command, script_unresolved): raw_path = raw_path.strip('"\'') if "://" in raw_path: continue - candidate = Path(raw_path) - local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() + try: + candidate = Path(raw_path) + local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() + except (ValueError, OSError) as error: + script_unresolved.append(f"unresolved local path {raw_path!r}: {error}") + continue try: local.relative_to(target.resolve()) except ValueError: @@ -330,6 +353,7 @@ def _check_package_scripts(target, sink): f"package script {name} points to missing local file {raw_path}", "package.json [scripts]", ) + unresolved.extend(f"package script {name}: {item}" for item in script_unresolved) def _read_source_pin(): @@ -367,7 +391,7 @@ def audit_path(target_path, source_pin=None): if has_pyproject: _check_pyproject_scripts(target, sink) if has_package: - _check_package_scripts(target, sink) + _check_package_scripts(target, sink, document["hmmm"]) document["surfaces"] = surfaces document["findings"] = sink.finalize() diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index 5bb6e09..d0c51ff 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -18,6 +18,7 @@ from . import _msdmd_universal as _canonical_msdmd from . import boundary +from . import ratios, source_boundaries SHEBANG_RE = re.compile(r"^#!.*$") _RATIOS_LINE_RE = re.compile(r"^(?:#|//|--|%|;|!|'|\*>)\s*ratios:\s*(.+?)\s*$") @@ -35,7 +36,7 @@ def _comment_markers() -> dict[str, str]: return dict(markers) if isinstance(markers, dict) else {} -def source_text(text: str, marker: str | None) -> str: +def source_text(text: str, marker: str | None, path: Path | None = None) -> str: """Return source text with complete generated NARRATIVE/RATIOS metadata removed. Trailing blank lines are normalized because RATIOS placement already removes @@ -44,24 +45,11 @@ def source_text(text: str, marker: str | None) -> str: if marker is None: return text - start = f"{marker} === NARRATIVE ===" - end = f"{marker} === END NARRATIVE ===" lines = text.splitlines() - kept: list[str] = [] - index = 0 - - while index < len(lines): - raw = lines[index] - if raw.rstrip() == start: - close = index + 1 - while close < len(lines) and lines[close].rstrip() != end: - close += 1 - if close < len(lines): - index = close + 1 - continue - if not _RATIOS_LINE_RE.match(raw.rstrip()): - kept.append(raw) - index += 1 + adapter = ratios.default_adapter_for(path) if path is not None else None + bookends, narrative = source_boundaries.metadata_indices(lines, marker, adapter) + excluded = bookends | narrative + kept = [line for index, line in enumerate(lines) if index not in excluded] while kept and not kept[-1].strip(): kept.pop() @@ -134,7 +122,7 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: item.hmmm.append("metadata-excluding source hash unavailable; mutation disabled") return item - stable_encoded = source_text(text, marker).encode("utf-8") + stable_encoded = source_text(text, marker, path).encode("utf-8") item.sha256 = hashlib.sha256(stable_encoded).hexdigest() first_line = text.splitlines()[0].rstrip() if text.splitlines() else "" @@ -142,12 +130,13 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: item.shebang = first_line if marker is not None: - for raw_line in text.splitlines(): - if _RATIOS_LINE_RE.match(raw_line.rstrip()): - item.ratios_lines.append(raw_line.rstrip()) + lines = text.splitlines() + bookends, narrative_indices = source_boundaries.metadata_indices(lines, marker, ratios.default_adapter_for(path)) + item.ratios_lines = [lines[index].rstrip() for index in sorted(bookends)] parser = _msdmd_parser() for name in _block_names(text, marker): - entries = parser.parse_text(text, name, marker) + block_text = "\n".join(lines[index] for index in sorted(narrative_indices)) if name == "NARRATIVE" else text + entries = parser.parse_text(block_text, name, marker) item.msdmd_blocks[name] = entries item.narrative_entries = item.msdmd_blocks.get("NARRATIVE", []) else: diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index ac3ff21..1c943d1 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -84,7 +84,7 @@ def _apply( if narrate: result = narrative.narrate_file( ev, - evidence.source_text(original_text, ev.marker), + evidence.source_text(original_text, ev.marker, path), provider_list, now, ) @@ -98,14 +98,23 @@ def _apply( if block_changed: file_changes.append(f"{ev.path}:narrative") - values = engine.compute(path, evidence.source_text(new_text, ev.marker)) + values = engine.compute(path, evidence.source_text(new_text, ev.marker, path)) new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) if ratio_changed: file_changes.append(f"{ev.path}:ratios") if new_text != original_text: try: - msdmd_writer.write_text_safely(path, new_text, ev.encoding) + if path.is_symlink() or boundary.assert_inside(root, path) != path or path.read_bytes() != raw: + unresolved[ev.path] = "source changed during examination; mutation skipped" + continue + msdmd_writer.write_text_safely(path, new_text, ev.encoding, expected_raw=raw) + except msdmd_writer.SourceChangedError: + unresolved[ev.path] = "source changed during examination; mutation skipped" + continue + except OSError as exc: + unresolved[ev.path] = f"source unavailable before write; mutation skipped: {exc}" + continue except UnicodeEncodeError: unresolved[ev.path] = f"generated text cannot use {ev.encoding}; mutation skipped" continue diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 59df970..5192e33 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -8,8 +8,11 @@ from __future__ import annotations from pathlib import Path +import os +import tempfile from . import ratios +from . import source_boundaries NARRATIVE_BLOCK = "NARRATIVE" @@ -22,24 +25,11 @@ def _block_lines(marker: str, entry: dict[str, str]) -> list[str]: return lines -def _without_narrative_lines(text: str, marker: str) -> list[str]: +def _without_narrative_lines(text: str, marker: str, adapter=None) -> list[str]: """Remove complete NARRATIVE blocks without creating phantom blank lines.""" - start = f"{marker} === {NARRATIVE_BLOCK} ===" - end = f"{marker} === END {NARRATIVE_BLOCK} ===" lines = text.splitlines() - kept: list[str] = [] - index = 0 - - while index < len(lines): - if lines[index].rstrip() == start: - close = index + 1 - while close < len(lines) and lines[close].rstrip() != end: - close += 1 - if close < len(lines): - index = close + 1 - continue - kept.append(lines[index]) - index += 1 + _, indices = source_boundaries.metadata_indices(lines, marker, adapter) + kept = [line for index, line in enumerate(lines) if index not in indices] while kept and not kept[-1].strip(): kept.pop() @@ -57,10 +47,10 @@ def upsert_narrative( path: Path | None = None, ) -> tuple[str, bool]: """Replace NARRATIVE blocks without crossing the protected opening boundary.""" - lines = _without_narrative_lines(text, marker) + adapter = ratios.RatiosEngine().adapter_for(path) if path is not None else None + lines = _without_narrative_lines(text, marker, adapter) block = "\n".join(_block_lines(marker, entry)) - adapter = ratios.RatiosEngine().adapter_for(path) if path is not None else None insert_at = ratios.opening_index(lines, adapter) ratios_prefix = f"{marker} ratios:" if insert_at < len(lines) and lines[insert_at].lstrip().startswith(ratios_prefix): @@ -71,7 +61,11 @@ def upsert_narrative( return new_text, new_text != text -def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8") -> None: +class SourceChangedError(RuntimeError): + """The live source no longer matches the inventoried bytes.""" + + +def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, expected_raw: bytes | None = None) -> None: """Preserve the source encoding and mode; encode before opening for writing.""" encoded = new_text.encode(encoding) mode = None @@ -79,6 +73,16 @@ def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8") -> Non mode = path.stat().st_mode & 0o777 except OSError: pass - path.write_bytes(encoded) - if mode is not None: - path.chmod(mode) + temporary = None + try: + with tempfile.NamedTemporaryFile(mode="wb", dir=path.parent, prefix=".examiner-", delete=False) as stream: + temporary = Path(stream.name) + stream.write(encoded) + if mode is not None: + temporary.chmod(mode) + if expected_raw is not None and (path.is_symlink() or path.read_bytes() != expected_raw): + raise SourceChangedError("source changed before metadata publication") + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) diff --git a/src/pubskill_lib/ratios.py b/src/pubskill_lib/ratios.py index 7df538a..3d523c5 100644 --- a/src/pubskill_lib/ratios.py +++ b/src/pubskill_lib/ratios.py @@ -27,6 +27,7 @@ import re from pathlib import Path +from . import source_boundaries RATIO_IDS = ("loc_comments", "imports_exports", "calls_definitions") SHEBANG_RE = re.compile(r"^#!.*$") @@ -135,10 +136,11 @@ def render_ratios_line(marker: str, values: dict[str, str]) -> str: return f"{marker} ratios: {body}" -def strip_ratios_lines(text: str, marker: str) -> list[str]: - """Return the file's lines with every ratios line removed.""" - line_re = _ratios_line_re(marker) - return [line for line in text.splitlines() if not line_re.match(line.rstrip())] +def strip_ratios_lines(text: str, marker: str, adapter=None) -> list[str]: + """Remove only the reserved bookends, preserving source-literal contents.""" + lines = text.splitlines() + indices, _ = source_boundaries.metadata_indices(lines, marker, adapter) + return [line for index, line in enumerate(lines) if index not in indices] def opening_index(lines: list[str], adapter: LanguageRatioAdapter | None) -> int: @@ -148,11 +150,7 @@ def opening_index(lines: list[str], adapter: LanguageRatioAdapter | None) -> int adapter, the default rule applies: a shebang stays first and the seal follows it. """ - if adapter is not None: - protected = adapter.opening_boundary(lines) - else: - protected = [0] if lines and SHEBANG_RE.match(lines[0].rstrip()) else [] - return max(protected, default=-1) + 1 + return source_boundaries.opening_index(lines, adapter) def place_ratios( @@ -166,8 +164,7 @@ def place_ratios( Returns ``(new_text, changed)``. Existing ratios lines are removed and re-placed. The closing line is the last non-blank line. """ - line_re = _ratios_line_re(marker) - lines = [raw for raw in text.splitlines() if not line_re.match(raw.rstrip())] + lines = strip_ratios_lines(text, marker, adapter) opening = render_ratios_line(marker, values) lines.insert(opening_index(lines, adapter), opening) diff --git a/src/pubskill_lib/source_boundaries.py b/src/pubskill_lib/source_boundaries.py new file mode 100644 index 0000000..be76b25 --- /dev/null +++ b/src/pubskill_lib/source_boundaries.py @@ -0,0 +1,40 @@ +"""Identify examiner metadata only at its reserved source placement boundaries. + +Fence-shaped text elsewhere remains source data, including inside multiline +strings. Entry parsing remains owned by the packaged canonical msdmd parser. +""" +from __future__ import annotations + +import re + + +def opening_index(lines, adapter=None): + if adapter is not None: + protected = adapter.opening_boundary(lines) + else: + protected = [0] if lines and lines[0].startswith("#!") else [] + return max(protected, default=-1) + 1 + + +def metadata_indices(lines, marker, adapter=None): + """Return reserved RATIOS indices and one complete opening NARRATIVE span.""" + ratio_line = re.compile(rf"^{re.escape(marker)}\s*ratios:\s*.+?\s*$") + ratios = set() + opening = opening_index(lines, adapter) + if opening < len(lines) and ratio_line.fullmatch(lines[opening]): + ratios.add(opening) + opening += 1 + closing = len(lines) - 1 + while closing >= 0 and not lines[closing].strip(): + closing -= 1 + if closing >= 0 and ratio_line.fullmatch(lines[closing]): + ratios.add(closing) + narrative = set() + if opening < len(lines) and lines[opening].rstrip() == f"{marker} === NARRATIVE ===": + for index in range(opening + 1, len(lines)): + if not lines[index].startswith(marker): + break + if lines[index].rstrip() == f"{marker} === END NARRATIVE ===": + narrative.update(range(opening, index + 1)) + break + return ratios, narrative diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 04f7464..9332c42 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -47,6 +47,67 @@ def test_blank_model_override_uses_provider_default(self): class NarrativeBoundaryTests(unittest.TestCase): + def test_fence_shaped_literal_data_remains_source(self): + class FakeProvider: + name, model = "fake", "model-1" + def chat(self, system, user): + return "Stores a literal string." + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "literal.py" + original = 'payload = """\n# === NARRATIVE ===\n# id: literal_data\n# summary: alpha\n# === END NARRATIVE ===\n# ratios: loc_comments=1:2 imports_exports=3:4 calls_definitions=5:6\n"""\n' + path.write_text(original) + before = evidence.read_evidence(root, path) + self.assertEqual([], before.narrative_entries) + path.write_text(original.replace("alpha", "beta")) + self.assertNotEqual(before.sha256, evidence.read_evidence(root, path).sha256) + path.write_text(original) + examine._apply(root, [before], [FakeProvider()], True) + namespace = {} + exec(compile(path.read_bytes(), str(path), "exec"), namespace) + expected = {} + exec(compile(original, str(path), "exec"), expected) + self.assertEqual(expected["payload"], namespace["payload"]) + after = evidence.read_evidence(root, path) + self.assertEqual(before.sha256, after.sha256) + self.assertEqual(1, len(after.narrative_entries)) + first = path.read_bytes() + examine._apply(root, [after], [], False) + self.assertEqual(first, path.read_bytes()) + + def test_provider_cannot_overwrite_a_concurrent_source_edit(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "concurrent.py" + path.write_text("print('old')\n") + concurrent = b"print('concurrent edit')\n" + class EditingProvider: + name, model = "fake", "model-1" + def chat(self, system, user): + path.write_bytes(concurrent) + return "Prints old." + _, report = examine._apply(root, [evidence.read_evidence(root, path)], [EditingProvider()], True) + self.assertEqual(concurrent, path.read_bytes()) + self.assertEqual([], report["changed"]) + self.assertIn("concurrent.py", report["hmmm"]) + + def test_failed_publication_preserves_source_and_external_hardlinks(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + original = b"original\n" + path.write_bytes(original) + alias = Path(tmp) / "external.py" + alias.hardlink_to(path) + with patch("pubskill_lib.msdmd_writer.os.replace", side_effect=OSError("publication failed")): + with self.assertRaises(OSError): + msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + self.assertEqual(original, path.read_bytes()) + self.assertEqual([], list(Path(tmp).glob(".examiner-*"))) + msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + self.assertEqual(original, alias.read_bytes()) + self.assertEqual(b"new\n", path.read_bytes()) + def test_apply_preserves_packaged_canonical_parser(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -336,6 +397,23 @@ def test_inspector_endpoint_requires_equals_in_node_24(self): # Official Node v24.15.0 attempts to load 9229 as the entry file here. self.assertEqual(["9229"], list(audit._local_script_targets("node --inspect 9229 app.js"))) + def test_node_inspect_subcommand_and_malformed_urls(self): + self.assertEqual(["missing.js"], list(audit._local_script_targets("node inspect missing.js"))) + self.assertEqual(["missing.js"], list(audit._local_script_targets("node inspect --port=9000 missing.js"))) + self.assertEqual([], list(audit._local_script_targets("node inspect localhost:9229"))) + self.assertEqual([], list(audit._local_script_targets("node inspect -p 1234"))) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "inspect").mkdir() + (root / "package.json").write_text(json.dumps({"scripts": { + "inspect": "node inspect missing.js", + "nul": "node --entry-url file:///tmp/%00.js", + "raw-nul": "node bad\u0000name.js", + }})) + document = audit.audit_path(root, "pin") + self.assertTrue(any("missing local file missing.js" in f["claim"] for f in document["findings"])) + self.assertEqual(2, sum("NUL-containing" in item for item in document["hmmm"])) + def test_quoted_segments_and_entrypoint_urls(self): self.assertEqual(["first.js", "second.js"], list(audit._local_script_targets("node --test-name-pattern 'a|b' first.js && node second.js"))) self.assertEqual([], list(audit._local_script_targets("node --entry-url 'data:text/javascript,console.log(1);'"))) From c65f40337dfae79b3e2c1bc12065dff7051204e0 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 03:20:17 +0000 Subject: [PATCH 08/23] Preserve source inodes and publish without replacing competing edits --- .gitignore | 2 ++ README.md | 11 ++++++ src/pubskill_lib/audit.py | 32 ++++++++--------- src/pubskill_lib/boundary.py | 2 +- src/pubskill_lib/examine.py | 17 +++++---- src/pubskill_lib/msdmd_writer.py | 62 +++++++++++++++++++++++--------- tests/test_repairs.py | 58 +++++++++++++++++++++++++++--- 7 files changed, 139 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 45125d0..78d3764 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ __pycache__/ .env .env.* !.env.example + +.examiner-originals-*/ diff --git a/README.md b/README.md index 4088b43..775bd3c 100644 --- a/README.md +++ b/README.md @@ -112,3 +112,14 @@ MPL-2.0, same as skill-lib. Changes to MPL-covered files must be published. ## Canon Do not add skills here first. Add them in skill-lib, mark them appropriately, pin the SHA in `SOURCE.md`, then propagate the public slice. + +Source updates preserve the original inode in a private `.examiner-originals-*` +directory beside the file, recorded under `preserved_sources` in the apply report. +These recovery directories are excluded from examiner inventory and should not be +committed. Publication briefly withdraws the old name, then creates the updated +name only if it remains absent; it never replaces a competing live file. A +collision or observed write to the retained original records `hmmm`. Already-open +writers can still change the retained original after the operation; stop editors +and generators before applying, then inspect recovery files before removing them. +This protocol preserves bytes; it does not claim a transactional edit shared with +uncooperative writers or uninterrupted availability to concurrent readers. diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index ab414e5..adf8a82 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -199,7 +199,7 @@ def _check_pyproject_scripts(target, sink): ) -def _shell_segments(command): +def _shell_segments(command, separators=";&|\n"): """Split direct shell commands while retaining quoted/escaped separators.""" start, quote, escaped = 0, None, False for index, character in enumerate(command): @@ -212,7 +212,7 @@ def _shell_segments(command): quote = None elif character in {"'", '"'}: quote = character - elif character in ";&|\n": + elif character in separators: yield command[start:index] start = index + 1 yield command[start:] @@ -227,6 +227,8 @@ def _entrypoint_target(token, entry_url, unresolved=None): raise ValueError("unsupported file URL authority") if parsed.scheme not in {"", "file"}: return None + if re.search(r"%(?![0-9a-fA-F]{2})|%(?:2[fF]|5[cC])", parsed.path): + raise ValueError("invalid or unsupported encoded URL path separator") target = unquote(parsed.path, errors="strict") if not target or "\0" in target: raise ValueError("empty or NUL-containing path") @@ -251,27 +253,25 @@ def _local_script_targets(command, unresolved=None): tokens = shlex.split(segment) except ValueError: continue + raw_words = [word for word in _shell_segments(segment, " \t\r") if word] + while tokens and raw_words and re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", raw_words[0]): + tokens.pop(0) + raw_words.pop(0) if not tokens or tokens[0] not in LOCAL_SCRIPT_INTERPRETERS: continue interpreter = tokens[0] non_file_modes = NON_FILE_MODES[interpreter] entry_url = False + inspecting = False index = 1 while index < len(tokens): token = tokens[index] - if interpreter == "node" and token == "inspect": - arguments = tokens[index + 1:] - if not arguments: - break - target = arguments[0] - if re.fullmatch(r"[^:]+:\d+", target) or (len(arguments) == 2 and target == "-p" and arguments[1].isdigit()): - break # Attach to an existing debugger/process, not a file. - if re.fullmatch(r"--port=\d+", target): - target = arguments[1] if len(arguments) > 1 else "" - target = _entrypoint_target(target, False, unresolved) - if target is not None: - yield target - break + if interpreter == "node" and token == "inspect" and not inspecting: + inspecting = True + index += 1 + continue + if inspecting and re.fullmatch(r"[^:]+:\d+", token): + break # Remote debugger attachment. if interpreter == "node" and token in {"--entry-url", "--experimental-entry-url"}: entry_url = True index += 1 @@ -334,8 +334,6 @@ def _check_package_scripts(target, sink, unresolved): script_unresolved = [] for raw_path in _local_script_targets(command, script_unresolved): raw_path = raw_path.strip('"\'') - if "://" in raw_path: - continue try: candidate = Path(raw_path) local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() diff --git a/src/pubskill_lib/boundary.py b/src/pubskill_lib/boundary.py index 3335390..74be335 100644 --- a/src/pubskill_lib/boundary.py +++ b/src/pubskill_lib/boundary.py @@ -62,7 +62,7 @@ def iter_files(root: Path, skip: set[str] | None = None) -> list[Path]: skip = set(skip if skip is not None else DEFAULT_SKIP) found: list[Path] = [] for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = sorted(d for d in dirnames if d not in skip) + dirnames[:] = sorted(d for d in dirnames if d not in skip and not d.startswith(".examiner-originals-")) for name in sorted(filenames): path = Path(dirpath) / name try: diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index 1c943d1..b06cc1b 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -31,14 +31,16 @@ def _canonical_artifact(path: Path) -> bool: def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: - supported = [ev for ev in evidence_list if ev.marker is not None and not _canonical_artifact(Path(ev.path))] + engine = ratios.RatiosEngine() + supported = [ev for ev in evidence_list if ev.marker is not None and ev.encoding is not None and engine.adapter_for(Path(ev.path)) is not None and not _canonical_artifact(Path(ev.path))] return { "root": str(root), "files": len(evidence_list), "supported_files": len(supported), "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(Path(ev.path))], "unsupported": [ - {"path": ev.path, "hmmm": ev.hmmm} for ev in evidence_list if ev.marker is None + {"path": ev.path, "hmmm": ev.hmmm or ["no safe metrics/write adapter"]} + for ev in evidence_list if ev not in supported and not _canonical_artifact(Path(ev.path)) ], "ratios_missing": [ev.path for ev in supported if not ev.ratios_lines], "narrative_present": [ev.path for ev in supported if ev.narrative_entries], @@ -55,6 +57,7 @@ def _apply( changed: list[str] = [] unresolved: dict[str, str] = {} preserved_authority: list[str] = [] + preserved_sources: dict[str, str] = {} now = datetime.now(timezone.utc).isoformat() engine = ratios.RatiosEngine() @@ -67,6 +70,7 @@ def _apply( continue adapter = engine.adapter_for(path) if ev.marker is None or adapter is None or ev.encoding is None: + unresolved[ev.path] = "; ".join(ev.hmmm) or "no safe metrics/write adapter; mutation skipped" continue try: raw = path.read_bytes() @@ -108,9 +112,10 @@ def _apply( if path.is_symlink() or boundary.assert_inside(root, path) != path or path.read_bytes() != raw: unresolved[ev.path] = "source changed during examination; mutation skipped" continue - msdmd_writer.write_text_safely(path, new_text, ev.encoding, expected_raw=raw) - except msdmd_writer.SourceChangedError: - unresolved[ev.path] = "source changed during examination; mutation skipped" + original = msdmd_writer.write_text_safely(path, new_text, ev.encoding, expected_raw=raw) + preserved_sources[ev.path] = original.relative_to(root).as_posix() + except msdmd_writer.SourceChangedError as exc: + unresolved[ev.path] = str(exc) continue except OSError as exc: unresolved[ev.path] = f"source unavailable before write; mutation skipped: {exc}" @@ -122,7 +127,7 @@ def _apply( narratives[ev.path] = entry changed.extend(file_changes) - return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved, "preserved_authority": preserved_authority} + return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved, "preserved_authority": preserved_authority, "preserved_sources": preserved_sources} def main(argv: list[str] | None = None) -> int: diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 5192e33..092f34c 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -65,24 +65,52 @@ class SourceChangedError(RuntimeError): """The live source no longer matches the inventoried bytes.""" -def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, expected_raw: bytes | None = None) -> None: - """Preserve the source encoding and mode; encode before opening for writing.""" +def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, expected_raw: bytes | None = None) -> Path: + """Publish without replacing a live name; retain the original inode. + + There is a short absent-name interval. Publication uses link's atomic + no-replace guarantee. Open writers keep their original inode in recovery + storage, which is deliberately never deleted by this operation. + """ encoded = new_text.encode(encoding) - mode = None - try: - mode = path.stat().st_mode & 0o777 - except OSError: - pass - temporary = None + if path.is_symlink(): + raise SourceChangedError("source became a symlink; mutation skipped") + raw = path.read_bytes() if expected_raw is None else expected_raw + mode = path.stat().st_mode & 0o777 + # A fresh private directory prevents a preexisting recovery path from + # redirecting writes. The caller reports its path; inventory skips it. + recovery = Path(tempfile.mkdtemp(prefix=".examiner-originals-", dir=path.parent)) + original = recovery / "original" + candidate = recovery / "candidate" + candidate.write_bytes(encoded) + candidate.chmod(mode) + moved = False try: - with tempfile.NamedTemporaryFile(mode="wb", dir=path.parent, prefix=".examiner-", delete=False) as stream: - temporary = Path(stream.name) - stream.write(encoded) - if mode is not None: - temporary.chmod(mode) - if expected_raw is not None and (path.is_symlink() or path.read_bytes() != expected_raw): + if path.is_symlink() or path.read_bytes() != raw: raise SourceChangedError("source changed before metadata publication") - os.replace(temporary, path) + os.rename(path, original) + moved = True + if original.is_symlink() or original.read_bytes() != raw: + raise SourceChangedError(f"source changed during publication; preserved at {original}") + try: + os.link(candidate, path) # Atomic create-if-absent; never replace a competing edit. + except FileExistsError as error: + raise SourceChangedError(f"competing source preserved; prior inode at {original}") from error + if original.read_bytes() != raw: + raise SourceChangedError(f"open writer changed original inode; inspect preserved source at {original}") + return original + except BaseException as failure: + if moved: + try: + os.link(original, path, follow_symlinks=False) + except FileExistsError: + pass # Preserve the live name and the recovery inode independently. + except OSError as error: + raise SourceChangedError(f"source retained at {original}; restore failed: {error}") from error + if isinstance(failure, OSError): + raise OSError(f"publication failed; original retained at {original}: {failure}") from failure + raise finally: - if temporary is not None: - temporary.unlink(missing_ok=True) + candidate.unlink(missing_ok=True) + if not moved: + recovery.rmdir() diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 9332c42..a2e7770 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -92,20 +92,47 @@ def chat(self, system, user): self.assertEqual([], report["changed"]) self.assertIn("concurrent.py", report["hmmm"]) + def test_conditional_publication_preserves_competing_writes(self): + import os + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + original = b"original\n" + concurrent = b"concurrent\n" + path.write_bytes(original) + link = os.link + def competing_write(source, target, **kwargs): + if Path(source).name == "candidate": + path.write_bytes(concurrent) + return link(source, target, **kwargs) + with patch("pubskill_lib.msdmd_writer.os.link", side_effect=competing_write): + with self.assertRaises(msdmd_writer.SourceChangedError): + msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + self.assertEqual(concurrent, path.read_bytes()) + self.assertEqual([original], [p.read_bytes() for p in Path(tmp).glob(".examiner-originals-*/original")]) + def test_failed_publication_preserves_source_and_external_hardlinks(self): + import os with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "source.py" original = b"original\n" path.write_bytes(original) alias = Path(tmp) / "external.py" alias.hardlink_to(path) - with patch("pubskill_lib.msdmd_writer.os.replace", side_effect=OSError("publication failed")): + link = os.link + def fail_candidate(source, target, **kwargs): + if Path(source).name == "candidate": + raise OSError("publication failed") + return link(source, target, **kwargs) + with patch("pubskill_lib.msdmd_writer.os.link", side_effect=fail_candidate): with self.assertRaises(OSError): msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) self.assertEqual(original, path.read_bytes()) - self.assertEqual([], list(Path(tmp).glob(".examiner-*"))) - msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) - self.assertEqual(original, alias.read_bytes()) + with path.open("r+b") as writer: + recovery = msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + writer.write(b"late edit") + writer.truncate() + self.assertEqual(b"late edit", recovery.read_bytes()) + self.assertEqual(b"late edit", alias.read_bytes()) self.assertEqual(b"new\n", path.read_bytes()) def test_apply_preserves_packaged_canonical_parser(self): @@ -192,6 +219,8 @@ def test_apply_skips_parser_supported_language_without_safe_ratio_adapter(self): _, report = examine._apply(root, [ev], [], False) self.assertEqual(original, path.read_text(encoding="utf-8")) self.assertEqual([], report["changed"]) + self.assertIn("index.php", report["hmmm"]) + self.assertEqual(0, examine._plan(root, [ev])["supported_files"]) def test_apply_preserves_encoding_and_source_identity(self): @@ -397,6 +426,27 @@ def test_inspector_endpoint_requires_equals_in_node_24(self): # Official Node v24.15.0 attempts to load 9229 as the entry file here. self.assertEqual(["9229"], list(audit._local_script_targets("node --inspect 9229 app.js"))) + def test_assignment_prefixes_inspect_options_and_url_paths(self): + commands = ( + "NODE_ENV=production node missing.js", + "A='value with spaces' B=two node missing.js", + "node inspect --trace-warnings missing.js", + "node inspect --require preload.js missing.js", + "node inspect --port=9000 --require preload.js missing.js", + ) + for command in commands: + self.assertEqual(["missing.js"], list(audit._local_script_targets(command)), command) + self.assertEqual([], list(audit._local_script_targets("'A=literal-command' node missing.js"))) + for operand in ("file:missing%2Fpart.js", "file:missing%5Cpart.js", "file:missing%ZZ.js"): + unresolved = [] + self.assertEqual([], list(audit._local_script_targets("node --entry-url " + operand, unresolved))) + self.assertTrue(unresolved) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "package.json").write_text(json.dumps({"scripts": {"url": "node https://example.invalid/missing.js"}})) + claims = [item["claim"] for item in audit.audit_path(root, "pin")["findings"] if item["surface"] == "deps"] + self.assertTrue(any("missing local file https://example.invalid/missing.js" in claim for claim in claims), claims) + def test_node_inspect_subcommand_and_malformed_urls(self): self.assertEqual(["missing.js"], list(audit._local_script_targets("node inspect missing.js"))) self.assertEqual(["missing.js"], list(audit._local_script_targets("node inspect --port=9000 missing.js"))) From 08acfb0324c8d018ed3a502e9cf0b6e5e1727160 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 03:36:14 +0000 Subject: [PATCH 09/23] Enforce canonical placement and expose safe source recovery --- README.md | 11 +++++--- src/pubskill_lib/examine.py | 39 +++++++++++++++++++++----- src/pubskill_lib/msdmd_writer.py | 7 +++++ src/pubskill_lib/ratios.py | 7 +++++ tests/test_examine.py | 26 +++++++++++------- tests/test_idempotence.py | 2 +- tests/test_repairs.py | 47 ++++++++++++++++++-------------- 7 files changed, 96 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 775bd3c..7f4f03d 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,11 @@ OPENAI_BASE_URL / ANTHROPIC_BASE_URL # process environment only Multiple configured providers are attempted sequentially as fallback. Unsupported or not-faithfully-computable metrics remain `hmmm`; they are not guessed. -Python coding cookies and UTF-8 byte-order marks are preserved during source -mutation. If generated prose cannot be encoded in the source encoding, the file -is left intact and the apply report records `hmmm`. Existing narratives remain +UTF-8 byte-order marks are preserved during source mutation. Files whose protected +coding cookies conflict with the pinned canonical RATIOS placement are left intact +and reported as `hmmm`; this consumer cannot expand canonical placement rules. +If generated prose cannot be encoded in the source encoding, the file is also +left intact with `hmmm`. Existing narratives remain available in assembled documentation even when a file has no safe mutation adapter. ## License @@ -116,7 +118,8 @@ Do not add skills here first. Add them in skill-lib, mark them appropriately, pi Source updates preserve the original inode in a private `.examiner-originals-*` directory beside the file, recorded under `preserved_sources` in the apply report. These recovery directories are excluded from examiner inventory and should not be -committed. Publication briefly withdraws the old name, then creates the updated +committed. The required hard-link operations are probed before source is moved. +Publication briefly withdraws the old name, then creates the updated name only if it remains absent; it never replaces a competing live file. A collision or observed write to the retained original records `hmmm`. Already-open writers can still change the retained original after the operation; stop editors diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index b06cc1b..e177615 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -32,16 +32,30 @@ def _canonical_artifact(path: Path) -> bool: def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: engine = ratios.RatiosEngine() - supported = [ev for ev in evidence_list if ev.marker is not None and ev.encoding is not None and engine.adapter_for(Path(ev.path)) is not None and not _canonical_artifact(Path(ev.path))] + supported = [] + unsupported = [] + for ev in evidence_list: + if _canonical_artifact(Path(ev.path)): + continue + reason = "; ".join(ev.hmmm) + if ev.marker is None or ev.encoding is None or engine.adapter_for(Path(ev.path)) is None: + reason = reason or "no safe metrics/write adapter" + else: + try: + path = boundary.assert_inside(root, root / ev.path) + engine.place(path.read_bytes().decode(ev.encoding), ev.marker, {}, path) + except (OSError, UnicodeError, ratios.UnsupportedPlacementError) as error: + reason = str(error) + else: + supported.append(ev) + continue + unsupported.append({"path": ev.path, "hmmm": [reason]}) return { "root": str(root), "files": len(evidence_list), "supported_files": len(supported), "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(Path(ev.path))], - "unsupported": [ - {"path": ev.path, "hmmm": ev.hmmm or ["no safe metrics/write adapter"]} - for ev in evidence_list if ev not in supported and not _canonical_artifact(Path(ev.path)) - ], + "unsupported": unsupported, "ratios_missing": [ev.path for ev in supported if not ev.ratios_lines], "narrative_present": [ev.path for ev in supported if ev.narrative_entries], } @@ -82,6 +96,11 @@ def _apply( unresolved[ev.path] = f"source unavailable; mutation skipped: {exc}" continue + try: + engine.place(original_text, ev.marker, {}, path) + except ratios.UnsupportedPlacementError as error: + unresolved[ev.path] = str(error) + continue new_text = original_text file_changes: list[str] = [] entry = narratives.get(ev.path) @@ -103,7 +122,11 @@ def _apply( file_changes.append(f"{ev.path}:narrative") values = engine.compute(path, evidence.source_text(new_text, ev.marker, path)) - new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) + try: + new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) + except ratios.UnsupportedPlacementError as error: + unresolved[ev.path] = str(error) + continue if ratio_changed: file_changes.append(f"{ev.path}:ratios") @@ -171,11 +194,13 @@ def main(argv: list[str] | None = None) -> int: volume = assemble.assemble_docs(root, evidence_list, narratives, out_dir) if args.json: - print(json.dumps({"changed": report["changed"], "hmmm": report["hmmm"], "volume": str(volume)}, indent=2)) + print(json.dumps({**report, "volume": str(volume)}, indent=2)) else: print(f"applied: {len(report['changed'])} writes") for change in report["changed"]: print(f" {change}") + for path, original in report["preserved_sources"].items(): + print(f" preserved source: {path}: {original}") for path, reason in report["hmmm"].items(): print(f" hmmm: {path}: {reason}") print(f"assembled: {volume}") diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 092f34c..5318a5a 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -86,6 +86,12 @@ def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, exp candidate.chmod(mode) moved = False try: + # Both candidate publication and original restoration require links. + # Probe the same files/directory before withdrawing the live name. + for source in (candidate, path): + probe = recovery / "link-probe" + os.link(source, probe, follow_symlinks=False) + probe.unlink() if path.is_symlink() or path.read_bytes() != raw: raise SourceChangedError("source changed before metadata publication") os.rename(path, original) @@ -112,5 +118,6 @@ def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, exp raise finally: candidate.unlink(missing_ok=True) + (recovery / "link-probe").unlink(missing_ok=True) if not moved: recovery.rmdir() diff --git a/src/pubskill_lib/ratios.py b/src/pubskill_lib/ratios.py index 3d523c5..1e08484 100644 --- a/src/pubskill_lib/ratios.py +++ b/src/pubskill_lib/ratios.py @@ -153,6 +153,10 @@ def opening_index(lines: list[str], adapter: LanguageRatioAdapter | None) -> int return source_boundaries.opening_index(lines, adapter) +class UnsupportedPlacementError(ValueError): + """Protected source lines conflict with the pinned canonical seal boundary.""" + + def place_ratios( text: str, marker: str, @@ -176,6 +180,9 @@ def place_ratios( new_text = "\n".join(lines) if lines: new_text += "\n" + from . import _msdmd_universal + if _msdmd_universal.ratios_placement(new_text, marker) != (True, True): + raise UnsupportedPlacementError("protected source prologue conflicts with pinned canonical RATIOS placement; mutation skipped") return new_text, new_text != text diff --git a/tests/test_examine.py b/tests/test_examine.py index 541e1f5..6500ac1 100644 --- a/tests/test_examine.py +++ b/tests/test_examine.py @@ -1,3 +1,4 @@ +import json import os import shutil import subprocess @@ -87,16 +88,11 @@ def test_python_opening_boundary_respects_encoding_header(self): self.assertEqual([0, 1], adapter.opening_boundary(lines)) self.assertEqual(2, ratios.opening_index(lines, adapter)) - new, _ = ratios.place_ratios( - "\n".join(lines) + "\n", - "#", - {"loc_comments": "1:0", "imports_exports": "1:0", "calls_definitions": "0:0"}, - adapter, - ) - out = new.splitlines() - self.assertTrue(out[0].startswith("#!")) - self.assertIn("coding", out[1]) - self.assertTrue(out[2].startswith("# ratios:")) + with self.assertRaises(ratios.UnsupportedPlacementError): + ratios.place_ratios( + "\n".join(lines) + "\n", "#", + {"loc_comments": "1:0", "imports_exports": "1:0", "calls_definitions": "0:0"}, adapter, + ) def test_find_internal_dependencies_python(self): from pubskill_lib import ratios_adapters @@ -169,10 +165,20 @@ def test_dry_run_reports_without_writing(self): self.assertEqual(0, result.returncode, result.stderr) self.assertEqual(before, (self.root / "tool.py").read_text()) + def test_json_apply_reports_recovery_paths(self): + result = self._run("--apply", "--json") + self.assertEqual(0, result.returncode, result.stderr) + report = json.loads(result.stdout) + self.assertTrue(report["preserved_sources"]) + for relative in report["preserved_sources"].values(): + self.assertTrue((self.root / relative).is_file()) + def test_apply_writes_ratios_and_assembles_docs(self): shell_before = (self.root / "run.sh").read_bytes() result = self._run("--apply", "--out", "docs/examiner") self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("preserved source:", result.stdout) + self.assertIn(".examiner-originals-", result.stdout) tool = (self.root / "tool.py").read_text().splitlines() self.assertTrue(tool[0].startswith("#!")) diff --git a/tests/test_idempotence.py b/tests/test_idempotence.py index 170a1f8..1aaed4d 100644 --- a/tests/test_idempotence.py +++ b/tests/test_idempotence.py @@ -19,7 +19,6 @@ def chat(self, system, user): source = root / "tool.py" source.write_text( "#!/usr/bin/env python3\n" - "# -*- coding: utf-8 -*-\n" "print('hi')\n", encoding="utf-8", ) @@ -28,6 +27,7 @@ def chat(self, system, user): _, first_report = examine._apply(root, first_evidence, [FakeProvider()], True) first_output = source.read_text(encoding="utf-8") self.assertTrue(first_report["changed"]) + self.assertEqual((True, True), evidence._canonical_msdmd.ratios_placement(first_output)) second_evidence = evidence.inventory(root) self.assertEqual(1, len(second_evidence)) diff --git a/tests/test_repairs.py b/tests/test_repairs.py index a2e7770..9e79d92 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -101,7 +101,7 @@ def test_conditional_publication_preserves_competing_writes(self): path.write_bytes(original) link = os.link def competing_write(source, target, **kwargs): - if Path(source).name == "candidate": + if Path(source).name == "candidate" and Path(target) == path: path.write_bytes(concurrent) return link(source, target, **kwargs) with patch("pubskill_lib.msdmd_writer.os.link", side_effect=competing_write): @@ -120,7 +120,7 @@ def test_failed_publication_preserves_source_and_external_hardlinks(self): alias.hardlink_to(path) link = os.link def fail_candidate(source, target, **kwargs): - if Path(source).name == "candidate": + if Path(source).name == "candidate" and Path(target) == path: raise OSError("publication failed") return link(source, target, **kwargs) with patch("pubskill_lib.msdmd_writer.os.link", side_effect=fail_candidate): @@ -223,30 +223,35 @@ def test_apply_skips_parser_supported_language_without_safe_ratio_adapter(self): self.assertEqual(0, examine._plan(root, [ev])["supported_files"]) - def test_apply_preserves_encoding_and_source_identity(self): - class FakeProvider: - name, model = "fake", "model-1" - def chat(self, system, user): - assert "café" in user - return "Prints café." - - for encoding in ("latin-1", "utf-8-sig"): + def test_encoding_prologues_stay_intact_when_canonical_placement_cannot_close(self): + for encoding, prefix in (("latin-1", "# coding: latin-1\n"), ("utf-8-sig", "#!/usr/bin/env python3\n# coding: utf-8\n")): with self.subTest(encoding=encoding), tempfile.TemporaryDirectory() as tmp: root = Path(tmp) path = root / "tool.py" - text = "# coding: " + ("utf-8" if encoding == "utf-8-sig" else encoding) + "\nprint('café')\n" - path.write_bytes(text.encode(encoding)) + raw = (prefix + "print('café')\n").encode(encoding) + path.write_bytes(raw) before = evidence.read_evidence(root, path) - examine._apply(root, [before], [FakeProvider()], True) - after = evidence.read_evidence(root, path) - self.assertEqual(before.sha256, after.sha256) - self.assertFalse(narrative.is_stale(after.narrative_entries[0], after.sha256)) - self.assertIn("café", path.read_bytes().decode(encoding)) - compile(path.read_bytes(), str(path), "exec") - first = path.read_bytes() - _, report = examine._apply(root, [after], [], False) - self.assertEqual(first, path.read_bytes()) + _, report = examine._apply(root, [before], [], False) + self.assertEqual(raw, path.read_bytes()) self.assertEqual([], report["changed"]) + self.assertIn("canonical RATIOS", report["hmmm"]["tool.py"]) + self.assertEqual(0, examine._plan(root, [before])["supported_files"]) + compile(path.read_bytes(), str(path), "exec") + # Encoding fidelity remains independently checked at the writer. + msdmd_writer.write_text_safely(path, prefix + "print('café updated')\n", encoding, expected_raw=raw) + compile(path.read_bytes(), str(path), "exec") + self.assertIn("café updated", path.read_bytes().decode(encoding)) + + def test_missing_hardlink_support_never_withdraws_live_source(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + original = b"original\n" + path.write_bytes(original) + with patch("pubskill_lib.msdmd_writer.os.link", side_effect=OSError("unsupported")): + with self.assertRaises(OSError): + msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + self.assertEqual(original, path.read_bytes()) + self.assertEqual([], list(Path(tmp).glob(".examiner-originals-*"))) def test_unrepresentable_narrative_does_not_truncate_source(self): class FakeProvider: From 22193d982d06c5f1cda72c36811801cae47b4ff6 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 03:39:13 +0000 Subject: [PATCH 10/23] Keep recovery CLI regressions valid after fixture annotation --- tests/test_examine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_examine.py b/tests/test_examine.py index 6500ac1..669b8b1 100644 --- a/tests/test_examine.py +++ b/tests/test_examine.py @@ -166,6 +166,7 @@ def test_dry_run_reports_without_writing(self): self.assertEqual(before, (self.root / "tool.py").read_text()) def test_json_apply_reports_recovery_paths(self): + (self.root / "recovery_probe.py").write_text("print('fresh source')\n") result = self._run("--apply", "--json") self.assertEqual(0, result.returncode, result.stderr) report = json.loads(result.stdout) @@ -174,6 +175,7 @@ def test_json_apply_reports_recovery_paths(self): self.assertTrue((self.root / relative).is_file()) def test_apply_writes_ratios_and_assembles_docs(self): + (self.root / "recovery_probe.py").write_text("print('fresh source')\n") shell_before = (self.root / "run.sh").read_bytes() result = self._run("--apply", "--out", "docs/examiner") self.assertEqual(0, result.returncode, result.stderr) From f0f4224580e4cc8bd22f18afd076b967f39768a1 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:02:58 +0000 Subject: [PATCH 11/23] Close audit context gaps and preserve source metadata and stale evidence --- README.md | 8 ++++ src/pubskill_lib/audit.py | 64 ++++++++++++++++++++++----- src/pubskill_lib/examine.py | 21 ++++++--- src/pubskill_lib/msdmd_writer.py | 32 +++++++++++--- tests/test_repairs.py | 74 ++++++++++++++++++++++++++++++++ 5 files changed, 179 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 7f4f03d..0188fd3 100644 --- a/README.md +++ b/README.md @@ -126,3 +126,11 @@ writers can still change the retained original after the operation; stop editors and generators before applying, then inspect recovery files before removing them. This protocol preserves bytes; it does not claim a transactional edit shared with uncooperative writers or uninterrupted availability to concurrent readers. + +Source publication currently requires Linux inode metadata support. Ownership, +permission bits, ACL/xattr/security-label bytes are copied and compared before +publication; an unavailable operation leaves the source intact with `hmmm`. +The generated volume uses a new source inventory after application, so preserved +concurrent edits can mark their older narratives stale. +Direct-script audit is intentionally bounded: unsupported commands, malformed +quoting, and working-directory transitions remain visible as `hmmm`. diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index adf8a82..d8e2d56 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -31,12 +31,20 @@ PIN_PATTERN = re.compile(r"`([0-9a-f]{40})`") LOCAL_SCRIPT_INTERPRETERS = {"node", "python", "python3", "bash", "sh"} NON_FILE_MODES = { - "node": {"-e", "--eval", "-p", "--print", "--run"}, - "python": {"-c", "-m"}, - "python3": {"-c", "-m"}, - "bash": {"-c"}, - "sh": {"-c"}, + "node": {"-e", "--eval", "-p", "--print", "--run", "-h", "--help", "-v", "--version", "--v8-options", "--completion-bash"}, + "python": {"-c", "-m", "-h", "-?", "--help", "-V", "--version", "--help-env", "--help-xoptions", "--help-all"}, + "python3": {"-c", "-m", "-h", "-?", "--help", "-V", "--version", "--help-env", "--help-xoptions", "--help-all"}, + "bash": {"-c", "--help", "--version"}, + "sh": {"-c", "--help", "--version"}, } +BOOLEAN_OPTIONS = { + "node": {"--trace-warnings", "--inspect", "--inspect-brk", "--inspect-wait", "--watch", "--test", "--no-warnings", "--enable-source-maps", "--experimental-strip-types", "--experimental-transform-types", "--abort-on-uncaught-exception", "--check", "--interactive", "-c", "-i"}, + "python": {"-" + character for character in "bBdEiIOPqRsSuvx"}, + "python3": {"-" + character for character in "bBdEiIOPqRsSuvx"}, + "bash": {"-" + character for character in "abefhkmnptuvxBCEHPTlirs"} | {"+" + character for character in "abefhkmnptuvxBCEHPTlirs"} | {"--debugger", "--dump-po-strings", "--dump-strings", "--noprofile", "--norc", "--posix", "--restricted", "--verbose", "--login"}, + "sh": {"-" + character for character in "aefnuvxCImps"} | {"+" + character for character in "aefnuvxCImps"}, +} + VALUE_OPTIONS = { "python": {"-W", "-X", "--check-hash-based-pycs"}, "python3": {"-W", "-X", "--check-hash-based-pycs"}, @@ -201,8 +209,13 @@ def _check_pyproject_scripts(target, sink): def _shell_segments(command, separators=";&|\n"): """Split direct shell commands while retaining quoted/escaped separators.""" - start, quote, escaped = 0, None, False + start, quote, escaped, comment = 0, None, False, False for index, character in enumerate(command): + if comment: + if character == "\n": + comment = False + start = index + 1 + continue if escaped: escaped = False elif character == "\\" and quote != "'": @@ -212,10 +225,14 @@ def _shell_segments(command, separators=";&|\n"): quote = None elif character in {"'", '"'}: quote = character + elif character == "#" and (index == 0 or command[index - 1] in " \t\r\n;&|()"): + yield command[start:index] + comment = True elif character in separators: yield command[start:index] start = index + 1 - yield command[start:] + if not comment: + yield command[start:] def _entrypoint_target(token, entry_url, unresolved=None): @@ -246,20 +263,36 @@ def _local_script_targets(command, unresolved=None): Python -W/-X, Bash -o/-O and startup files, and common Node value options consume their arguments; attached values and -- delimiters are supported. """ + cwd_unknown = False for segment in _shell_segments(command): if not segment.strip(): continue try: tokens = shlex.split(segment) - except ValueError: + except ValueError as error: + if unresolved is not None: + unresolved.append(f"unparseable package script: {error}") continue raw_words = [word for word in _shell_segments(segment, " \t\r") if word] while tokens and raw_words and re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", raw_words[0]): tokens.pop(0) raw_words.pop(0) - if not tokens or tokens[0] not in LOCAL_SCRIPT_INTERPRETERS: + if not tokens: + continue + interpreter = Path(tokens[0]).name + if interpreter in {"cd", "pushd", "popd"}: + cwd_unknown = True + if unresolved is not None: + unresolved.append("working-directory change is outside direct-script audit scope") + continue + if interpreter not in LOCAL_SCRIPT_INTERPRETERS: + if unresolved is not None: + unresolved.append(f"command is outside direct interpreter audit scope: {tokens[0]!r}") + continue + if cwd_unknown: + if unresolved is not None: + unresolved.append(f"script target after working-directory change is unresolved: {segment.strip()!r}") continue - interpreter = tokens[0] non_file_modes = NON_FILE_MODES[interpreter] entry_url = False inspecting = False @@ -288,6 +321,12 @@ def _local_script_targets(command, unresolved=None): index += 2 continue if token.startswith("-") or (interpreter in {"bash", "sh"} and token.startswith("+")): + if token.startswith("--"): + option = token.split("=", 1)[0] + if option not in BOOLEAN_OPTIONS[interpreter] and option not in VALUE_OPTIONS[interpreter] and not (inspecting and re.fullmatch(r"--port=\d+", token)): + if unresolved is not None: + unresolved.append(f"interpreter option arity is unresolved: {token!r}") + break # Short options may be clustered or carry an attached argument. non_file = False if not token.startswith("--"): @@ -302,6 +341,11 @@ def _local_script_targets(command, unresolved=None): if position == len(token) - 1: index += 1 break + if token[0] + option not in BOOLEAN_OPTIONS[interpreter]: + if unresolved is not None: + unresolved.append(f"interpreter option arity is unresolved: {token!r}") + non_file = True + break if non_file: break index += 1 diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index e177615..b7c2926 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -26,8 +26,14 @@ from . import ratios -def _canonical_artifact(path: Path) -> bool: - return path.name == "_msdmd_universal.py" and path.parent.name == "pubskill_lib" +def _canonical_artifact(root: Path, path: Path) -> bool: + candidate = path if path.is_absolute() else root / path + try: + return (candidate.relative_to(root).as_posix() == "src/pubskill_lib/_msdmd_universal.py" + and not candidate.is_symlink() + and candidate.read_bytes() == Path(evidence._canonical_msdmd.__file__).read_bytes()) + except (OSError, ValueError): + return False def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: @@ -35,7 +41,7 @@ def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: supported = [] unsupported = [] for ev in evidence_list: - if _canonical_artifact(Path(ev.path)): + if _canonical_artifact(root, Path(ev.path)): continue reason = "; ".join(ev.hmmm) if ev.marker is None or ev.encoding is None or engine.adapter_for(Path(ev.path)) is None: @@ -54,7 +60,7 @@ def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: "root": str(root), "files": len(evidence_list), "supported_files": len(supported), - "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(Path(ev.path))], + "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(root, Path(ev.path))], "unsupported": unsupported, "ratios_missing": [ev.path for ev in supported if not ev.ratios_lines], "narrative_present": [ev.path for ev in supported if ev.narrative_entries], @@ -79,7 +85,7 @@ def _apply( path = boundary.assert_inside(root, root / ev.path) if ev.narrative_entries: narratives[ev.path] = ev.narrative_entries[0] - if _canonical_artifact(path): + if _canonical_artifact(root, path): preserved_authority.append(ev.path) continue adapter = engine.adapter_for(path) @@ -190,6 +196,11 @@ def main(argv: list[str] | None = None) -> int: return 3 narratives, report = _apply(root, evidence_list, provider_list, args.narrate) + # Rendering observes live source after every write/skip, so an old summary + # cannot retain a current marker after a concurrent edit was preserved. + evidence_list = evidence.inventory(root) + narratives = {ev.path: ev.narrative_entries[0] for ev in evidence_list if ev.narrative_entries} + report["narrated"] = len(narratives) out_dir = boundary.assert_inside(root, root / args.out) volume = assemble.assemble_docs(root, evidence_list, narratives, out_dir) diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 5318a5a..3c4464a 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -9,6 +9,7 @@ from pathlib import Path import os +import sys import tempfile from . import ratios @@ -65,6 +66,27 @@ class SourceChangedError(RuntimeError): """The live source no longer matches the inventoried bytes.""" +def _inode_metadata(path: Path) -> tuple: + if not sys.platform.startswith("linux") or not hasattr(os, "listxattr"): + raise OSError("source inode metadata verification is unsupported on this platform") + info = path.stat(follow_symlinks=False) + attributes = {name: os.getxattr(path, name, follow_symlinks=False) + for name in os.listxattr(path, follow_symlinks=False)} + return info.st_uid, info.st_gid, info.st_mode & 0o7777, attributes + + +def _copy_inode_metadata(path: Path, metadata: tuple) -> None: + uid, gid, mode, attributes = metadata + os.chown(path, uid, gid, follow_symlinks=False) + path.chmod(mode) + for name in set(os.listxattr(path, follow_symlinks=False)) - attributes.keys(): + os.removexattr(path, name, follow_symlinks=False) + for name, value in attributes.items(): + os.setxattr(path, name, value, follow_symlinks=False) + if _inode_metadata(path) != metadata: + raise OSError("source inode metadata cannot be preserved exactly") + + def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, expected_raw: bytes | None = None) -> Path: """Publish without replacing a live name; retain the original inode. @@ -76,27 +98,27 @@ def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, exp if path.is_symlink(): raise SourceChangedError("source became a symlink; mutation skipped") raw = path.read_bytes() if expected_raw is None else expected_raw - mode = path.stat().st_mode & 0o777 + metadata = _inode_metadata(path) # A fresh private directory prevents a preexisting recovery path from # redirecting writes. The caller reports its path; inventory skips it. recovery = Path(tempfile.mkdtemp(prefix=".examiner-originals-", dir=path.parent)) original = recovery / "original" candidate = recovery / "candidate" - candidate.write_bytes(encoded) - candidate.chmod(mode) moved = False try: + candidate.write_bytes(encoded) + _copy_inode_metadata(candidate, metadata) # Both candidate publication and original restoration require links. # Probe the same files/directory before withdrawing the live name. for source in (candidate, path): probe = recovery / "link-probe" os.link(source, probe, follow_symlinks=False) probe.unlink() - if path.is_symlink() or path.read_bytes() != raw: + if path.is_symlink() or path.read_bytes() != raw or _inode_metadata(path) != metadata: raise SourceChangedError("source changed before metadata publication") os.rename(path, original) moved = True - if original.is_symlink() or original.read_bytes() != raw: + if original.is_symlink() or original.read_bytes() != raw or _inode_metadata(original) != metadata: raise SourceChangedError(f"source changed during publication; preserved at {original}") try: os.link(candidate, path) # Atomic create-if-absent; never replace a competing edit. diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 9e79d92..89cb6c3 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -135,6 +135,68 @@ def fail_candidate(source, target, **kwargs): self.assertEqual(b"late edit", alias.read_bytes()) self.assertEqual(b"new\n", path.read_bytes()) + def test_unrelated_similarly_named_source_is_not_canonical_authority(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name in ("other/pubskill_lib/_msdmd_universal.py", "src/pubskill_lib/_msdmd_universal.py"): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("print('ordinary user code')\n") + item = evidence.read_evidence(root, path) + self.assertEqual(1, examine._plan(root, [item])["supported_files"]) + _, report = examine._apply(root, [item], [], False) + self.assertTrue(report["changed"]) + self.assertEqual([], report["preserved_authority"]) + + def test_candidate_setup_failure_cleans_recovery_storage(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + path.write_text("original\n") + for operation in ("write_bytes", "chmod"): + with patch.object(Path, operation, side_effect=OSError("quota")): + with self.assertRaises(OSError): + msdmd_writer.write_text_safely(path, "new\n") + self.assertEqual("original\n", path.read_text()) + self.assertEqual([], list(Path(tmp).glob(".examiner-originals-*"))) + + def test_publication_preserves_inode_metadata_or_refuses_to_move(self): + import os + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + path.write_text("original\n") + path.chmod(0o751) + os.setxattr(path, "user.pubskill_test", b"retained") + original_metadata = msdmd_writer._inode_metadata(path) + msdmd_writer.write_text_safely(path, "new\n") + self.assertEqual(original_metadata, msdmd_writer._inode_metadata(path)) + with patch("pubskill_lib.msdmd_writer.os.setxattr", side_effect=OSError("metadata denied")): + with self.assertRaises(OSError): + msdmd_writer.write_text_safely(path, "third\n") + self.assertEqual("new\n", path.read_text()) + self.assertEqual(original_metadata, msdmd_writer._inode_metadata(path)) + + def test_assembled_narrative_is_stale_after_preserving_concurrent_edit(self): + import contextlib, io + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "source.py" + original = "print('old')\n" + path.write_text(original) + ev = evidence.read_evidence(root, path) + decorated, _ = msdmd_writer.upsert_narrative(original, "#", {"id": "old_narrative", "summary": "Old summary", "evidence_sha256": ev.sha256}, path) + path.write_text(decorated) + class EditingProvider: + name, model = "fake", "model" + def chat(self, system, user): + path.write_text(path.read_text().replace("print('old')", "print('edited')")) + return "Generated stale summary" + with patch("pubskill_lib.providers.configured_providers", return_value=[EditingProvider()]), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(0, examine.main(["--repo", str(root), "--apply", "--narrate"])) + output = (root / "docs/examiner/EXAMINER.md").read_text() + self.assertIn("Old summary", output) + self.assertIn("> stale:", output) + self.assertIn("print('edited')", path.read_text()) + def test_apply_preserves_packaged_canonical_parser(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -486,6 +548,18 @@ def test_quoted_segments_and_entrypoint_urls(self): self.assertTrue(any("escapes repository via /definitely/missing.js" in claim for claim in claims)) self.assertTrue(any("missing local file missing.js" in claim for claim in claims)) + def test_exit_comments_paths_and_unresolved_shell_context(self): + for command in ("python --help missing.py", "python -uV missing.py", "node --version missing.js", "bash --help missing.sh"): + self.assertEqual([], list(audit._local_script_targets(command)), command) + self.assertEqual(["real.js"], list(audit._local_script_targets("node real.js # disabled && node missing.js"))) + self.assertEqual(["real.js", "next.js"], list(audit._local_script_targets("node real.js # disabled && node missing.js\nnode next.js"))) + self.assertEqual(["missing.js"], list(audit._local_script_targets("/usr/bin/node missing.js"))) + self.assertEqual(["missing.py"], list(audit._local_script_targets("./venv/bin/python missing.py"))) + for command in ("node 'missing.js", "cd frontend && node build.js", "node --unknown-option value missing.js", "unknown-runner missing.js"): + unresolved = [] + self.assertEqual([], list(audit._local_script_targets(command, unresolved)), command) + self.assertTrue(unresolved, command) + def test_non_object_package_manifest_is_target_defect(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From 03aa395dbf879f76cfc735316c47194cdedb19f2 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:10:18 +0000 Subject: [PATCH 12/23] Keep unresolved shell expansion contexts out of literal path findings --- README.md | 4 +++- src/pubskill_lib/audit.py | 32 ++++++++++++++++++++++++++++++++ tests/test_repairs.py | 3 ++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0188fd3..2f7cf13 100644 --- a/README.md +++ b/README.md @@ -133,4 +133,6 @@ publication; an unavailable operation leaves the source intact with `hmmm`. The generated volume uses a new source inventory after application, so preserved concurrent edits can mark their older narratives stale. Direct-script audit is intentionally bounded: unsupported commands, malformed -quoting, and working-directory transitions remain visible as `hmmm`. +quoting, shell expansions/control syntax, and working-directory transitions remain +visible as `hmmm`. Later commands after an unresolved shell context inherit that +uncertainty; the tool does not guess their working directory or entrypoint. diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index d8e2d56..c212531 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -235,6 +235,30 @@ def _shell_segments(command, separators=";&|\n"): yield command[start:] +def _shell_context_gap(segment): + """Refuse expansion/control syntax that this literal-path audit cannot resolve.""" + quote, escaped = None, False + for character in segment: + if escaped: + if character == "\n": + return "shell line continuation is outside literal-path audit scope" + escaped = False + continue + if character == "\\" and quote != "'": + escaped = True + elif quote == "'": + if character == "'": + quote = None + elif character in "$`" or (quote is None and character in "*?[]{}()<>~"): + return "shell expansion or control syntax is outside literal-path audit scope" + elif quote: + if character == quote: + quote = None + elif character in {"'", '"'}: + quote = character + return None + + def _entrypoint_target(token, entry_url, unresolved=None): try: target = token @@ -267,9 +291,16 @@ def _local_script_targets(command, unresolved=None): for segment in _shell_segments(command): if not segment.strip(): continue + gap = _shell_context_gap(segment) + if gap: + cwd_unknown = True + if unresolved is not None: + unresolved.append(gap) + continue try: tokens = shlex.split(segment) except ValueError as error: + cwd_unknown = True if unresolved is not None: unresolved.append(f"unparseable package script: {error}") continue @@ -286,6 +317,7 @@ def _local_script_targets(command, unresolved=None): unresolved.append("working-directory change is outside direct-script audit scope") continue if interpreter not in LOCAL_SCRIPT_INTERPRETERS: + cwd_unknown = True if unresolved is not None: unresolved.append(f"command is outside direct interpreter audit scope: {tokens[0]!r}") continue diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 89cb6c3..95c3893 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -555,7 +555,8 @@ def test_exit_comments_paths_and_unresolved_shell_context(self): self.assertEqual(["real.js", "next.js"], list(audit._local_script_targets("node real.js # disabled && node missing.js\nnode next.js"))) self.assertEqual(["missing.js"], list(audit._local_script_targets("/usr/bin/node missing.js"))) self.assertEqual(["missing.py"], list(audit._local_script_targets("./venv/bin/python missing.py"))) - for command in ("node 'missing.js", "cd frontend && node build.js", "node --unknown-option value missing.js", "unknown-runner missing.js"): + self.assertEqual(["$literal.js"], list(audit._local_script_targets("node '$literal.js'"))) + for command in ("node 'missing.js", "cd frontend && node build.js", "node --unknown-option value missing.js", "unknown-runner missing.js", "node $SCRIPT", "node *.js", "(cd frontend && node build.js)", "node < input.js", "node \\\n missing.js"): unresolved = [] self.assertEqual([], list(audit._local_script_targets(command, unresolved)), command) self.assertTrue(unresolved, command) From e5c5dee5eecd931f7b2042bea0bb3bf7be101a3b Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:15:35 +0000 Subject: [PATCH 13/23] Use declared boundary values for release tooling --- tools/build_release.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/build_release.py b/tools/build_release.py index 52c40e2..2eb20b0 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -7,8 +7,10 @@ # public_surface: python tools/build_release.py --out DIRECTORY # internal_surface: normalize_sdist, main # auth_boundary: none -# storage_boundary: temporary build directory and explicit output directory -# network_boundary: none; build dependencies must already be installed +# storage_boundary: write +# storage_notes: temporary build directory and explicit output directory +# network_boundary: none +# network_notes: build dependencies must already be installed # user_data_boundary: none # admin_only: false # tests: clean-install repository suite and two-build digest comparison documented in README From 8d351daa8e225dd06564d13799c135d87c7e2d11 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:24:33 +0000 Subject: [PATCH 14/23] Keep literal entrypoints visible around dynamic child arguments --- src/pubskill_lib/audit.py | 32 ++++++++++++++++++++------------ tests/test_repairs.py | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index c212531..37b558f 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -235,10 +235,10 @@ def _shell_segments(command, separators=";&|\n"): yield command[start:] -def _shell_context_gap(segment): - """Refuse expansion/control syntax that this literal-path audit cannot resolve.""" +def _shell_context_gap(segment, *, context_only=False): + """Identify unsupported syntax, separating word expansion from shell structure.""" quote, escaped = None, False - for character in segment: + for index, character in enumerate(segment): if escaped: if character == "\n": return "shell line continuation is outside literal-path audit scope" @@ -249,8 +249,11 @@ def _shell_context_gap(segment): elif quote == "'": if character == "'": quote = None - elif character in "$`" or (quote is None and character in "*?[]{}()<>~"): - return "shell expansion or control syntax is outside literal-path audit scope" + elif quote is None and (character in "`{}()<>" or segment[index:index + 2] == "$("): + return "shell control syntax is outside literal-path audit scope" + elif not context_only and (character in "$`" or (quote is None and + (character in "*?[]" or (character == "~" and (index == 0 or segment[index - 1].isspace()))))): + return "shell word expansion is outside literal-path audit scope" elif quote: if character == quote: quote = None @@ -292,11 +295,10 @@ def _local_script_targets(command, unresolved=None): if not segment.strip(): continue gap = _shell_context_gap(segment) - if gap: - cwd_unknown = True - if unresolved is not None: - unresolved.append(gap) - continue + if gap and unresolved is not None: + unresolved.append(gap) + prior_cwd_unknown = cwd_unknown + cwd_unknown = cwd_unknown or bool(_shell_context_gap(segment, context_only=True)) try: tokens = shlex.split(segment) except ValueError as error: @@ -310,6 +312,9 @@ def _local_script_targets(command, unresolved=None): raw_words.pop(0) if not tokens: continue + if _shell_context_gap(raw_words[0]): + cwd_unknown = True # A dynamic command could resolve to a shell builtin. + continue interpreter = Path(tokens[0]).name if interpreter in {"cd", "pushd", "popd"}: cwd_unknown = True @@ -321,7 +326,7 @@ def _local_script_targets(command, unresolved=None): if unresolved is not None: unresolved.append(f"command is outside direct interpreter audit scope: {tokens[0]!r}") continue - if cwd_unknown: + if prior_cwd_unknown: if unresolved is not None: unresolved.append(f"script target after working-directory change is unresolved: {segment.strip()!r}") continue @@ -330,6 +335,8 @@ def _local_script_targets(command, unresolved=None): inspecting = False index = 1 while index < len(tokens): + if _shell_context_gap(" ".join(raw_words[:index + 1])): + break token = tokens[index] if interpreter == "node" and token == "inspect" and not inspecting: inspecting = True @@ -342,7 +349,8 @@ def _local_script_targets(command, unresolved=None): index += 1 continue if token == "--": - if index + 1 < len(tokens) and tokens[index + 1] != "-": + if (index + 1 < len(tokens) and tokens[index + 1] != "-" + and not _shell_context_gap(" ".join(raw_words[:index + 2]))): target = _entrypoint_target(tokens[index + 1], entry_url, unresolved) if target is not None: yield target diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 95c3893..f26e7f3 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -548,6 +548,26 @@ def test_quoted_segments_and_entrypoint_urls(self): self.assertTrue(any("escapes repository via /definitely/missing.js" in claim for claim in claims)) self.assertTrue(any("missing local file missing.js" in claim for claim in claims)) + def test_word_expansions_preserve_independent_literal_entrypoints(self): + for command, expected in ( + ('node missing.js "$ARG"', ["missing.js"]), + ('python missing.py "$ARG"', ["missing.py"]), + ('node -- missing.js "$ARG"', ["missing.js"]), + ('node build~backup.js', ["build~backup.js"]), + ('node $SCRIPT && node missing.js', ["missing.js"]), + ('node *.js && node missing.js', ["missing.js"]), + ('node "$SCRIPT" && node missing.js', ["missing.js"]), + ('node missing.js "$(pwd)"', ["missing.js"]), + ('node ~/script.js', []), + ('node -- $SCRIPT', []), + ('node $(cd ..; node hidden.js; pwd) && node uncertain.js', []), + ): + with self.subTest(command=command): + gaps = [] + self.assertEqual(list(audit._local_script_targets(command, gaps)), expected) + if "$" in command or "*" in command or "~/" in command: + self.assertTrue(gaps) + def test_exit_comments_paths_and_unresolved_shell_context(self): for command in ("python --help missing.py", "python -uV missing.py", "node --version missing.js", "bash --help missing.sh"): self.assertEqual([], list(audit._local_script_targets(command)), command) From 71c636941bd696dfba7f257eeb65942c84f7e4a1 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:45:19 +0000 Subject: [PATCH 15/23] Close shell operand and undecodable evidence hash gaps --- src/pubskill_lib/audit.py | 67 ++++++++++++++++++++++++++++++++---- src/pubskill_lib/evidence.py | 10 +++++- tests/test_repairs.py | 53 ++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 8 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 37b558f..1fce1be 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -237,13 +237,16 @@ def _shell_segments(command, separators=";&|\n"): def _shell_context_gap(segment, *, context_only=False): """Identify unsupported syntax, separating word expansion from shell structure.""" - quote, escaped = None, False + quote, escaped, word_start = None, False, 0 for index, character in enumerate(segment): if escaped: if character == "\n": return "shell line continuation is outside literal-path audit scope" escaped = False continue + if quote is None and character.isspace(): + word_start = index + 1 + continue if character == "\\" and quote != "'": escaped = True elif quote == "'": @@ -252,7 +255,9 @@ def _shell_context_gap(segment, *, context_only=False): elif quote is None and (character in "`{}()<>" or segment[index:index + 2] == "$("): return "shell control syntax is outside literal-path audit scope" elif not context_only and (character in "$`" or (quote is None and - (character in "*?[]" or (character == "~" and (index == 0 or segment[index - 1].isspace()))))): + (character in "*?[]" or (character == "~" and (index == word_start or + (re.match(r"[A-Za-z_][A-Za-z0-9_]*=", segment[word_start:index]) + and segment[index - 1] in "=:")))))): return "shell word expansion is outside literal-path audit scope" elif quote: if character == quote: @@ -262,6 +267,50 @@ def _shell_context_gap(segment, *, context_only=False): return None +def _fixed_word_arity(raw): + """Bounded proof that a supported option value remains one shell argument.""" + quote, escaped = None, False + for index, character in enumerate(raw): + if escaped: + escaped = False + continue + if character == "\\" and quote != "'": + escaped = True + elif quote == "'": + if character == "'": + quote = None + elif quote == '"': + if character == '"': + quote = None + elif character == "$": + # Ordinary quoted scalar expansions have fixed arity. Positional + # arrays and complex parameter/substitution forms stay unresolved. + tail = raw[index:] + if not re.match(r"\$(?:[A-Za-z_][A-Za-z0-9_]*|[0-9*#?$!]|\{[A-Za-z_][A-Za-z0-9_]*\})", tail): + return False + elif character == "`": + return False + elif character in {"'", '"'}: + quote = character + elif character in "$`*?[]{}()<>" or character.isspace(): + return False + return quote is None and not escaped + + +def _attached_option_value(token, interpreter): + if token.startswith("--"): + return "=" in token and token.split("=", 1)[0] in VALUE_OPTIONS[interpreter] + if not token.startswith(("-", "+")): + return False + for position, character in enumerate(token[1:], start=1): + option = token[0] + character + if option in VALUE_OPTIONS[interpreter]: + return position < len(token) - 1 + if option not in BOOLEAN_OPTIONS[interpreter]: + return False + return False + + def _entrypoint_target(token, entry_url, unresolved=None): try: target = token @@ -335,9 +384,10 @@ def _local_script_targets(command, unresolved=None): inspecting = False index = 1 while index < len(tokens): - if _shell_context_gap(" ".join(raw_words[:index + 1])): - break token = tokens[index] + if _shell_context_gap(raw_words[index]): + if not _attached_option_value(token, interpreter) or not _fixed_word_arity(raw_words[index]): + break if interpreter == "node" and token == "inspect" and not inspecting: inspecting = True index += 1 @@ -350,7 +400,7 @@ def _local_script_targets(command, unresolved=None): continue if token == "--": if (index + 1 < len(tokens) and tokens[index + 1] != "-" - and not _shell_context_gap(" ".join(raw_words[:index + 2]))): + and not _shell_context_gap(raw_words[index + 1])): target = _entrypoint_target(tokens[index + 1], entry_url, unresolved) if target is not None: yield target @@ -358,6 +408,8 @@ def _local_script_targets(command, unresolved=None): if token == "-" or token.split("=", 1)[0] in non_file_modes: break if token in VALUE_OPTIONS[interpreter]: + if index + 1 < len(raw_words) and not _fixed_word_arity(raw_words[index + 1]): + break index += 2 continue if token.startswith("-") or (interpreter in {"bash", "sh"} and token.startswith("+")): @@ -374,11 +426,13 @@ def _local_script_targets(command, unresolved=None): if interpreter in {"bash", "sh"}: modes.add("s") # Read commands from stdin. for position, option in enumerate(token[1:], start=1): - if token[0] == "-" and option in modes: + if (token[0] == "-" and option in modes) or (interpreter == "bash" and option == "s"): non_file = True break if token[0] + option in VALUE_OPTIONS[interpreter]: if position == len(token) - 1: + if index + 1 < len(raw_words) and not _fixed_word_arity(raw_words[index + 1]): + non_file = True index += 1 break if token[0] + option not in BOOLEAN_OPTIONS[interpreter]: @@ -417,7 +471,6 @@ def _check_package_scripts(target, sink, unresolved): continue script_unresolved = [] for raw_path in _local_script_targets(command, script_unresolved): - raw_path = raw_path.strip('"\'') try: candidate = Path(raw_path) local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index d0c51ff..afac379 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -122,7 +122,15 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: item.hmmm.append("metadata-excluding source hash unavailable; mutation disabled") return item - stable_encoded = source_text(text, marker, path).encode("utf-8") + try: + stable_encoded = source_text(text, marker, path).encode("utf-8") + except UnicodeError as error: + item.sha256 = item.raw_sha256 + item.marker = None + item.encoding = None + item.hmmm.append(f"source encoding unresolved while hashing: {error}") + item.hmmm.append("metadata-excluding source hash unavailable; mutation disabled") + return item item.sha256 = hashlib.sha256(stable_encoded).hexdigest() first_line = text.splitlines()[0].rstrip() if text.splitlines() else "" diff --git a/tests/test_repairs.py b/tests/test_repairs.py index f26e7f3..0c6c072 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -379,6 +379,24 @@ def test_raw_sha256_hashes_literal_python_bytes_and_honors_cookie(self): self.assertEqual("#", item.marker) self.assertFalse(item.hmmm) + def test_unencodable_stable_source_hash_is_hmmm_without_mutation(self): + from contextlib import redirect_stdout + import io + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "surrogate.py" + raw = b"# coding: unicode_escape\n# " + bytes((92,)) + b"ud800\n" + path.write_bytes(raw) + item = evidence.read_evidence(root, path) + self.assertEqual(hashlib.sha256(raw).hexdigest(), item.sha256) + self.assertIsNone(item.encoding) + self.assertIsNone(item.marker) + self.assertTrue(any("encoding unresolved while hashing" in text for text in item.hmmm)) + for options in ([], ["--apply"]): + with redirect_stdout(io.StringIO()): + self.assertEqual(0, examine.main(["--repo", str(root), "--json", *options])) + self.assertEqual(raw, path.read_bytes()) + def test_undecodable_non_python_source_is_hmmm_and_not_mutable(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -568,6 +586,41 @@ def test_word_expansions_preserve_independent_literal_entrypoints(self): if "$" in command or "*" in command or "~/" in command: self.assertTrue(gaps) + def test_fixed_arity_option_values_and_bash_stdin_modes(self): + for command, expected in ( + ('node --require "$PRELOAD" missing.js', ["missing.js"]), + ('node --require="$PRELOAD" missing.js', ["missing.js"]), + ('node -r"$PRELOAD" missing.js', ["missing.js"]), + ('python -W "$WARN" missing.py', ["missing.py"]), + ('python -uW"$WARN" missing.py', ["missing.py"]), + ('python -uW "$WARN" missing.py', ["missing.py"]), + ('node --require "$@" uncertain.js', []), + ('node --require $PRELOAD uncertain.js', []), + ('bash +s missing.sh', []), + ('bash +es missing.sh', []), + ): + with self.subTest(command=command): + self.assertEqual(expected, list(audit._local_script_targets(command))) + + def test_assignment_tildes_and_literal_filename_quotes(self): + for command, expected in ( + ('node foo=~/bar', []), + ('node foo=prefix:~/bar', []), + ('node foo-bar=~/bar', ["foo-bar=~/bar"]), + ('node build~backup.js', ["build~backup.js"]), + ): + gaps = [] + self.assertEqual(expected, list(audit._local_script_targets(command, gaps)), command) + self.assertEqual(not expected, bool(gaps), command) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "present.js").write_text("// ordinary name exists\n") + (root / "package.json").write_text(json.dumps({"scripts": {"quoted": "node " + "'" + '\"present.js\"' + "'"}})) + claims = [finding["claim"] for finding in audit.audit_path(root, "pin")["findings"]] + self.assertTrue(any('missing local file "present.js"' in claim for claim in claims), claims) + (root / '\"present.js\"').write_text("// exact quote-named file exists\n") + self.assertFalse(any("missing local file" in finding["claim"] for finding in audit.audit_path(root, "pin")["findings"])) + def test_exit_comments_paths_and_unresolved_shell_context(self): for command in ("python --help missing.py", "python -uV missing.py", "node --version missing.js", "bash --help missing.sh"): self.assertEqual([], list(audit._local_script_targets(command)), command) From 80a935a02b837fdd393a19e132eadbe9e9626722 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:57:33 +0000 Subject: [PATCH 16/23] Restrict assignment tilde expansion to its first separator or colons --- src/pubskill_lib/audit.py | 4 ++-- tests/test_repairs.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 1fce1be..65a1c5c 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -256,8 +256,8 @@ def _shell_context_gap(segment, *, context_only=False): return "shell control syntax is outside literal-path audit scope" elif not context_only and (character in "$`" or (quote is None and (character in "*?[]" or (character == "~" and (index == word_start or - (re.match(r"[A-Za-z_][A-Za-z0-9_]*=", segment[word_start:index]) - and segment[index - 1] in "=:")))))): + (re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=", segment[word_start:index]) + or (segment[index - 1] == ":" and re.match(r"[A-Za-z_][A-Za-z0-9_]*=", segment[word_start:index])))))))): return "shell word expansion is outside literal-path audit scope" elif quote: if character == quote: diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 0c6c072..c03c999 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -606,6 +606,7 @@ def test_assignment_tildes_and_literal_filename_quotes(self): for command, expected in ( ('node foo=~/bar', []), ('node foo=prefix:~/bar', []), + ('node entry=value=~/missing.js', ["entry=value=~/missing.js"]), ('node foo-bar=~/bar', ["foo-bar=~/bar"]), ('node build~backup.js', ["build~backup.js"]), ): From 1aeff0c050526bdf5958594215a6c5fbaa8777f9 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 05:53:04 +0000 Subject: [PATCH 17/23] Record the build interpreter in release provenance --- tools/build_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_release.py b/tools/build_release.py index 2eb20b0..2ac412b 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -109,7 +109,7 @@ def git(*args): raise ValueError(f"unexpected build artifact: {artifact.name}") hashes = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(out.iterdir())} doctrine = json.loads((root / "src/pubskill_lib/_source.json").read_text()) - manifest = {"schema": "pubskill-lib.release-manifest", "version": 1, "source_commit": commit, "source_tree": git("rev-parse", "HEAD^{tree}"), "source_date_epoch": epoch, "skill_lib_commit": doctrine["commit"], "build_toolchain": versions, "artifacts_sha256": hashes} + manifest = {"schema": "pubskill-lib.release-manifest", "version": 1, "source_commit": commit, "source_tree": git("rev-parse", "HEAD^{tree}"), "source_date_epoch": epoch, "skill_lib_commit": doctrine["commit"], "build_toolchain": versions, "build_python": sys.version, "artifacts_sha256": hashes} receipt = out / "release-manifest.json" receipt.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") hashes = dict(hashes) From 785b2d852b9a3a3f853c14bd631171a1efabea49 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 07:05:27 +0000 Subject: [PATCH 18/23] Ignore generated packaging output during release verification --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 78d3764..253fb17 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ __pycache__/ *.pyc *.egg-info/ +build/ +dist/ .env .env.* !.env.example From b9e20e82c23da3270d90f3c0bf55115c8b28edeb Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 07:21:13 +0000 Subject: [PATCH 19/23] Normalize wheel metadata across build environments --- .github/workflows/ci.yml | 10 ++++++++-- README.md | 5 ++++- tools/build_release.py | 20 ++++++++++++++++---- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 395c820..9884b54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,14 @@ jobs: run: | python -m venv /tmp/pubskill-build-venv /tmp/pubskill-build-venv/bin/python -m pip install -r requirements-build.txt - /tmp/pubskill-build-venv/bin/python tools/build_release.py --out /tmp/pubskill-release-a - /tmp/pubskill-build-venv/bin/python tools/build_release.py --out /tmp/pubskill-release-b + ( + umask 022 + /tmp/pubskill-build-venv/bin/python tools/build_release.py --out /tmp/pubskill-release-a + ) + ( + umask 077 + /tmp/pubskill-build-venv/bin/python tools/build_release.py --out /tmp/pubskill-release-b + ) diff /tmp/pubskill-release-a/SHA256SUMS /tmp/pubskill-release-b/SHA256SUMS /tmp/pubskill-wheel-venv/bin/python -m pip install --no-deps --force-reinstall /tmp/pubskill-release-a/*.whl mkdir /tmp/pubskill-replay diff --git a/README.md b/README.md index 2f7cf13..d7a1e9b 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,11 @@ python tools/build_release.py --out /tmp/pubskill-build-b diff /tmp/pubskill-build-a/SHA256SUMS /tmp/pubskill-build-b/SHA256SUMS ``` -The builder uses only committed source, normalizes source archive headers, and +The builder uses only committed source, normalizes source and wheel archive +headers, ordering, and permissions, and records source, doctrine, toolchain, and artifact digests in `release-manifest.json`. +CI compares builds under both 022 and 077 file-creation masks; wheel payloads +and their RECORD hashes remain unchanged by archive normalization. It does not publish. Before publication, install the exact wheel in a fresh venv, run the tests and fixture from the extracted sdist, and inspect a real consumer. The wheel retains its canonical skill-lib source pin without requiring a checkout. diff --git a/tools/build_release.py b/tools/build_release.py index 2ac412b..a3a13e0 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -5,7 +5,7 @@ # summary: builds normalized immutable wheel and sdist artifacts from a clean exact Git commit # owner: The Interdependency # public_surface: python tools/build_release.py --out DIRECTORY -# internal_surface: normalize_sdist, main +# internal_surface: normalize_sdist, normalize_wheel, main # auth_boundary: none # storage_boundary: write # storage_notes: temporary build directory and explicit output directory @@ -28,8 +28,8 @@ Run twice into separate empty directories and compare wheel/sdist SHA-256 values. The builder performs no publication. Clean-install and consumer gates are required -before publishing these bytes. Tar headers are normalized to the commit timestamp; -wheel timestamps use SOURCE_DATE_EPOCH. Source file contents are unchanged. +before publishing these bytes. Archive headers, order, and permissions are +normalized to the commit timestamp. Wheel payloads and RECORD are unchanged. """ from __future__ import annotations @@ -45,6 +45,8 @@ import sys import tarfile import tempfile +import time +import zipfile def normalize_sdist(path: Path, destination: Path, epoch: int) -> None: @@ -66,6 +68,16 @@ def normalize_sdist(path: Path, destination: Path, epoch: int) -> None: target.addfile(member) +def normalize_wheel(path: Path, destination: Path, epoch: int) -> None: + with zipfile.ZipFile(path) as source, zipfile.ZipFile(destination, "w") as target: + for member in sorted(source.infolist(), key=lambda item: item.filename): + normalized = zipfile.ZipInfo(member.filename, time.gmtime(epoch)[:6]) + normalized.create_system = 3 + mode = 0o40755 if member.is_dir() else 0o100755 if (member.external_attr >> 16) & 0o111 else 0o100644 + normalized.external_attr = (mode << 16) | (0x10 if member.is_dir() else 0) + target.writestr(normalized, source.read(member), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, required=True) @@ -104,7 +116,7 @@ def git(*args): if artifact.name.endswith(".tar.gz"): normalize_sdist(artifact, out / artifact.name, epoch) elif artifact.suffix == ".whl": - (out / artifact.name).write_bytes(artifact.read_bytes()) + normalize_wheel(artifact, out / artifact.name, epoch) else: raise ValueError(f"unexpected build artifact: {artifact.name}") hashes = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(out.iterdir())} From 9ad8f81497734abb687ca5f88b84448fd311ccb6 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 07:38:44 +0000 Subject: [PATCH 20/23] Bind release metadata and CI to the selected source commit --- .github/workflows/ci.yml | 1 + tools/build_release.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9884b54..c59f3b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ jobs: steps: - uses: actions/checkout@v7 with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - uses: actions/setup-python@v7 with: diff --git a/tools/build_release.py b/tools/build_release.py index a3a13e0..98f3c60 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -94,7 +94,7 @@ def git(*args): if any(out.iterdir()): raise SystemExit("release output directory must be empty") versions = {} - for requirement in (root / "requirements-build.txt").read_text().splitlines(): + for requirement in git("show", f"{commit}:requirements-build.txt").splitlines(): name, version = requirement.split("==") versions[name] = importlib.metadata.version(name) if versions[name] != version: @@ -120,8 +120,8 @@ def git(*args): else: raise ValueError(f"unexpected build artifact: {artifact.name}") hashes = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(out.iterdir())} - doctrine = json.loads((root / "src/pubskill_lib/_source.json").read_text()) - manifest = {"schema": "pubskill-lib.release-manifest", "version": 1, "source_commit": commit, "source_tree": git("rev-parse", "HEAD^{tree}"), "source_date_epoch": epoch, "skill_lib_commit": doctrine["commit"], "build_toolchain": versions, "build_python": sys.version, "artifacts_sha256": hashes} + doctrine = json.loads(git("show", f"{commit}:src/pubskill_lib/_source.json")) + manifest = {"schema": "pubskill-lib.release-manifest", "version": 1, "source_commit": commit, "source_tree": git("rev-parse", f"{commit}^{{tree}}"), "source_date_epoch": epoch, "skill_lib_commit": doctrine["commit"], "build_toolchain": versions, "build_python": sys.version, "artifacts_sha256": hashes} receipt = out / "release-manifest.json" receipt.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") hashes = dict(hashes) From 4001c46a777a216a4cf3eb0e368d0fafcacc6506 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 09:46:47 +0000 Subject: [PATCH 21/23] Filter source archive extraction before release builds --- tools/build_release.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/build_release.py b/tools/build_release.py index 98f3c60..7974cf1 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -104,11 +104,13 @@ def git(*args): source = temporary / "source" source.mkdir() archive = subprocess.check_output(["git", "-C", str(root), "archive", commit]) + if not hasattr(tarfile, "data_filter"): + raise SystemExit("release builds require Python with tarfile.data_filter support") with tarfile.open(fileobj=io.BytesIO(archive)) as tree: for member in tree.getmembers(): if member.name.startswith("/") or ".." in Path(member.name).parts or not (member.isfile() or member.isdir()): raise ValueError("unsafe source archive") - tree.extractall(source) + tree.extractall(source, filter="data") environment = dict(os.environ, SOURCE_DATE_EPOCH=str(epoch), PYTHONHASHSEED="0") environment.pop("PYTHONPATH", None) subprocess.run([sys.executable, "-m", "build", "--no-isolation", "--outdir", str(temporary / "dist"), str(source)], check=True, env=environment) From 24622c9e84eee1c4903382d35da84d33029e3b06 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 09:53:46 +0000 Subject: [PATCH 22/23] Bind release builds to the qualified zlib compressor --- .github/workflows/ci.yml | 3 ++- README.md | 6 +++++- tests/test_release_compressor.py | 14 ++++++++++++++ tools/build_release.py | 12 +++++++++++- 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 tests/test_release_compressor.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c59f3b7..e7f197f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,8 @@ jobs: /tmp/pubskill-wheel-venv/bin/python -c "from pubskill_lib import evidence; assert evidence._comment_markers()['.py'] == '#'" - name: Reproduce release artifacts and replay installed wheel run: | - python -m venv /tmp/pubskill-build-venv + python -m pip install uv==0.11.18 + uv venv --managed-python --python 3.11.15 /tmp/pubskill-build-venv /tmp/pubskill-build-venv/bin/python -m pip install -r requirements-build.txt ( umask 022 diff --git a/README.md b/README.md index d7a1e9b..62c363b 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ From the release's exact Git commit, install the pinned build tools and build into two empty directories: ```bash +python -m pip install uv==0.11.18 +uv venv --managed-python --python 3.11.15 /tmp/pubskill-build-env +. /tmp/pubskill-build-env/bin/activate python -m pip install -r requirements-build.txt python tools/build_release.py --out /tmp/pubskill-build-a python tools/build_release.py --out /tmp/pubskill-build-b @@ -46,7 +49,8 @@ diff /tmp/pubskill-build-a/SHA256SUMS /tmp/pubskill-build-b/SHA256SUMS The builder uses only committed source, normalizes source and wheel archive headers, ordering, and permissions, and -records source, doctrine, toolchain, and artifact digests in `release-manifest.json`. +requires zlib 1.3.1 at compile time and runtime, and records its identity along +with source, doctrine, toolchain, and artifact digests in `release-manifest.json`. CI compares builds under both 022 and 077 file-creation masks; wheel payloads and their RECORD hashes remain unchanged by archive normalization. It does not publish. Before publication, install the exact wheel in a fresh venv, diff --git a/tests/test_release_compressor.py b/tests/test_release_compressor.py new file mode 100644 index 0000000..2385b40 --- /dev/null +++ b/tests/test_release_compressor.py @@ -0,0 +1,14 @@ +"""Usage: python -m unittest discover -s tests. Reject an unqualified compressor.""" +import unittest +from unittest.mock import patch +from tools.build_release import check_compressor + + +class CompressorTest(unittest.TestCase): + def test_compressor_identity_is_enforced(self): + with patch("tools.build_release.zlib.ZLIB_VERSION", "1.3.1"), patch("tools.build_release.zlib.ZLIB_RUNTIME_VERSION", "1.3.1"): + self.assertEqual(check_compressor()["runtime_version"], "1.3.1") + for compile_version, runtime_version in (("1.3", "1.3.1"), ("1.3.1", "1.3")): + with patch("tools.build_release.zlib.ZLIB_VERSION", compile_version), patch("tools.build_release.zlib.ZLIB_RUNTIME_VERSION", runtime_version): + with self.assertRaisesRegex(RuntimeError, "require zlib"): + check_compressor() diff --git a/tools/build_release.py b/tools/build_release.py index 7974cf1..60d7dfa 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -47,6 +47,15 @@ import tempfile import time import zipfile +import zlib + + +def check_compressor() -> dict[str, str]: + expected = "1.3.1" + actual = {"implementation": "zlib", "compile_version": zlib.ZLIB_VERSION, "runtime_version": zlib.ZLIB_RUNTIME_VERSION} + if actual["compile_version"] != expected or actual["runtime_version"] != expected: + raise RuntimeError(f"release builds require zlib {expected} at compile time and runtime: {actual}") + return actual def normalize_sdist(path: Path, destination: Path, epoch: int) -> None: @@ -87,6 +96,7 @@ def git(*args): return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() if git("status", "--porcelain"): raise SystemExit("release build requires a clean Git checkout") + compressor = check_compressor() commit = git("rev-parse", "HEAD") epoch = int(git("show", "-s", "--format=%ct", commit)) out = args.out.resolve() @@ -123,7 +133,7 @@ def git(*args): raise ValueError(f"unexpected build artifact: {artifact.name}") hashes = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(out.iterdir())} doctrine = json.loads(git("show", f"{commit}:src/pubskill_lib/_source.json")) - manifest = {"schema": "pubskill-lib.release-manifest", "version": 1, "source_commit": commit, "source_tree": git("rev-parse", f"{commit}^{{tree}}"), "source_date_epoch": epoch, "skill_lib_commit": doctrine["commit"], "build_toolchain": versions, "build_python": sys.version, "artifacts_sha256": hashes} + manifest = {"schema": "pubskill-lib.release-manifest", "version": 1, "source_commit": commit, "source_tree": git("rev-parse", f"{commit}^{{tree}}"), "source_date_epoch": epoch, "skill_lib_commit": doctrine["commit"], "build_toolchain": versions, "build_python": sys.version, "compressor": compressor, "artifacts_sha256": hashes} receipt = out / "release-manifest.json" receipt.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") hashes = dict(hashes) From 0c305f049ed238de95fe516df6734f1dbee09a08 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 09:56:31 +0000 Subject: [PATCH 23/23] Install build dependencies with the selected uv environment --- .github/workflows/ci.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7f197f..68a4f16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: run: | python -m pip install uv==0.11.18 uv venv --managed-python --python 3.11.15 /tmp/pubskill-build-venv - /tmp/pubskill-build-venv/bin/python -m pip install -r requirements-build.txt + uv pip install --python /tmp/pubskill-build-venv/bin/python -r requirements-build.txt ( umask 022 /tmp/pubskill-build-venv/bin/python tools/build_release.py --out /tmp/pubskill-release-a diff --git a/README.md b/README.md index 62c363b..0ff80cd 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ into two empty directories: python -m pip install uv==0.11.18 uv venv --managed-python --python 3.11.15 /tmp/pubskill-build-env . /tmp/pubskill-build-env/bin/activate -python -m pip install -r requirements-build.txt +uv pip install --python /tmp/pubskill-build-env/bin/python -r requirements-build.txt python tools/build_release.py --out /tmp/pubskill-build-a python tools/build_release.py --out /tmp/pubskill-build-b diff /tmp/pubskill-build-a/SHA256SUMS /tmp/pubskill-build-b/SHA256SUMS