From 0753156388b2734b01611ce6d44920c80631dfd6 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Thu, 10 Sep 2026 17:41:55 -0700 Subject: [PATCH 01/39] repair: close audit defects before v0.2.0 --- .env.example | 7 + .github/workflows/ci.yml | 6 +- .gitignore | 3 + HANDOFF.md | 171 ++---------------- LICENSE | 147 ++++++++++++++- README.md | 61 +++---- SOURCE.md | 8 +- .../neglected-repo/expected-findings.json | 13 +- src/pubskill_lib.egg-info/PKG-INFO | 68 ------- src/pubskill_lib.egg-info/SOURCES.txt | 13 -- .../dependency_links.txt | 1 - src/pubskill_lib.egg-info/entry_points.txt | 2 - src/pubskill_lib.egg-info/top_level.txt | 1 - src/pubskill_lib/audit.py | 90 ++++++--- src/pubskill_lib/evidence.py | 93 +++++----- src/pubskill_lib/examine.py | 21 ++- src/pubskill_lib/msdmd_writer.py | 37 ++-- src/pubskill_lib/narrative.py | 12 +- src/pubskill_lib/providers.py | 45 +++-- tests/test_repairs.py | 117 ++++++++++++ 20 files changed, 494 insertions(+), 422 deletions(-) create mode 100644 .env.example delete mode 100644 src/pubskill_lib.egg-info/PKG-INFO delete mode 100644 src/pubskill_lib.egg-info/SOURCES.txt delete mode 100644 src/pubskill_lib.egg-info/dependency_links.txt delete mode 100644 src/pubskill_lib.egg-info/entry_points.txt delete mode 100644 src/pubskill_lib.egg-info/top_level.txt create mode 100644 tests/test_repairs.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..cac6e40 --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +# This file intentionally exists as an example only. Real credentials belong in +# your local .env (ignored) or process environment. +OPENAI_API_KEY= +OPENAI_MODEL= +ANTHROPIC_API_KEY= +ANTHROPIC_MODEL= +# Base URL overrides are process-environment-only and are not read from .env. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76792dd..293b507 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,11 +3,15 @@ on: [push, pull_request] jobs: test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.11', '3.12'] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: ${{ matrix.python-version }} - run: | python -m venv .venv . .venv/bin/activate diff --git a/.gitignore b/.gitignore index f03a7a0..45125d0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ __pycache__/ *.pyc *.egg-info/ +.env +.env.* +!.env.example diff --git a/HANDOFF.md b/HANDOFF.md index 304c993..01e8005 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,184 +1,51 @@ -# HANDOFF — populate pubskill-lib with clone utility - -This file is the work order. Execute in order. Do not skip to features. - -If the host is a VM, read `HANDOFF.vm.md` first. VM default: steps A–F only. No push, no tag, no skill-lib writes. +# HANDOFF — pubskill-lib v0.2 closure Owner repo: `The-Interdependency/pubskill-lib` Canon: `The-Interdependency/skill-lib` Current pin: see `SOURCE.md` Public claims: `README.md` -Agent contract: `AGENTS.md` -VM contract: `HANDOFF.vm.md` +Agent contract: `AGENTS.md` ## Goal -A stranger clones this repository, runs the commands in README.md, and gets a findings file for `examples/neglected-repo`. - -That is `v0.2.0`. Nothing else is the first tag. A VM receipt is not a tag. - -## Non-goals for this handoff - -- Do not port the full skill-lib catalog. -- Do not implement `--fix-one` until inspect works (that is `v0.3.0`). -- Do not host a SaaS. -- Do not rewrite msdmd. -- Do not add Way / UCNS / energy text to README. - -## Prerequisite in skill-lib - -Only if `WRITE_CANON=1`: - -1. Add `status` (`runnable` | `contract` | `org-only`) and optional `runner` to `skills.json`. -2. Mark only skills that execute in-repo as `runnable`. -3. Confirm `python -m unittest discover -s tests` passes on a clean skill-lib clone. - -VM default: skip this block. Implement the inspect CLI here. Follow repo-audit-repair classes. Do not invent a sixth class. - -Allowed classes: `defect`, `environment`, `external`, `policy`, `hmmm`. +A stranger clones this repository, installs it, runs its tests, audits `examples/neglected-repo`, and gets the expected findings file. That is the `v0.2.0` release gate. -## Target tree +## v0.2 public boundary -``` -pubskill-lib/ - README.md - AGENTS.md - SOURCE.md - HANDOFF.md - HANDOFF.vm.md - LICENSE - pyproject.toml - src/pubskill_lib/ - __init__.py - audit.py - schema.py - tests/ - test_audit_fixture.py - test_schema.py - examples/neglected-repo/ - README.md - pyproject.toml - .github/workflows/ci.yml - tests/test_dummy.py - expected-findings.json - .agents/skills/README.md - .agents/skills// - .github/workflows/ci.yml -``` - -## Step A — package skeleton - -Create `pyproject.toml` so `pip install -e .` works on Python 3.11+ with no extra native deps. - -Package name: `pubskill-lib` -Import name: `pubskill_lib` -Console optional: `pubskill-audit` - -Acceptance: - -```bash -python -m venv .venv && source .venv/bin/activate -python -m pip install -e . -python -c "import pubskill_lib" -``` - -## Step B — findings schema - -`findings.json` shape: - -```json -{ - "schema_version": 1, - "tool": "pubskill-lib", - "source_pin": "", - "target": { - "path": "", - "commit": "hmmm", - "remote": "hmmm", - "dirty": false - }, - "surfaces": [], - "findings": [ - { - "id": "F001", - "surface": "docs|ci|deps|identity|tests", - "claim": "", - "evidence": "", - "class": "defect", - "owner": "repository|environment|external|policy|hmmm", - "verified": false - } - ], - "hmmm": [] -} -``` - -`verified` defaults false. Absence of a finding is not health. - -## Step C — inspect CLI (no execute by default) +The inspect CLI accepts one local repository path: ```bash python -m pubskill_lib.audit PATH --out findings.json ``` -v0.2 inspect only: +It may inspect README files, `pyproject.toml`, `package.json`, and GitHub workflow text; record local git identity; flag missing local README targets; flag obvious test-workflow no-ops; and flag declared Python or direct local package-script entrypoints that do not exist. -- read README / pyproject / package.json / lockfiles / `.github/workflows/*` -- record identity if `.git` exists, else `hmmm` -- flag README links to missing local files -- flag workflows that claim tests but only `echo` -- flag missing advertised scripts -- do not install target deps -- do not run target tests +It does not clone remote URLs, select remote commits, install target dependencies, run target tests, repair target repositories, or expose `--fix-one`. Those capabilities require separate versioned work rather than implied v0.2 behavior. -Exit 0 if the tool ran. Do not exit nonzero just because the target repo is sick. Exit nonzero for tool/schema failures. +## Source provenance -## Step D — fixture +Vendored public skills must identify the same exact skill-lib SHA in both `SOURCE.md` and `.agents/skills/README.md`. Do not vendor from unpinned `main`. -`examples/neglected-repo` must contain at least three evidenced defects the CLI will see without `--run`: +## Credential boundary -1. README references a file that does not exist -2. CI workflow named like tests that does not invoke a test runner -3. a declared script or extra path that is missing +The examiner may read API keys and model names from `.env`, but provider base URL overrides are operator configuration and therefore come only from the process environment. Never allow an inspected repository's `.env` to redirect an ambient operator credential. -Write `expected-findings.json` from the CLI output, then lock it. - -Acceptance: - -```bash -python -m pubskill_lib.audit examples/neglected-repo --out /tmp/out.json -``` - -Compare on `id`, `class`, and `surface`. - -## Step E — vendor the slice - -Optional if time or disk is scarce. Prefer A–D first. - -From skill-lib at the SOURCE.md SHA, copy only `msdmd` and `repo-audit-repair` into `.agents/skills/` and write `.agents/skills/README.md` with the SHA. Do not copy the rest of skill-lib. - -## Step F — tests in this repo +## Release gate ```bash +python -m venv .venv && source .venv/bin/activate python -m pip install -e . python -m unittest discover -s tests python -m pubskill_lib.audit examples/neglected-repo --out /tmp/out.json ``` -A `.github/workflows/ci.yml` may be added. The VM does not push it unless `PUSH=1`. - -## Step G — close the public door - -Not a VM default. Requires `PUSH=1`. - -1. Rewrite README status table: inspect ships. -2. Tag `v0.2.0`. -3. Add this repo to skill-lib consumer list only with `WRITE_CANON=1`. +The first release tag is exactly `v0.2.0`. README may say the implementation is ready on `main` before that tag exists; it must not claim the release exists before the tag is published. -## Done / not done +## Generated metadata -Done: clean clone → install → unittest → audit fixture → findings.json. +`*.egg-info/` is generated build state. It is ignored and must not be tracked. Package/release verification regenerates metadata from `pyproject.toml`. -Not done: stars, SaaS, full catalog, architectural-drift theater, selling VERIFIED on the zip. +## hmmm -hmmm — if the fixture expected file was authored by hand and never produced by the CLI, the utility is still fake. +- Python 3.11 is part of the declared `>=3.11` support range but the current GitHub CI lane is Python 3.12 only. Add a 3.11 CI lane before treating cross-version support as independently witnessed. +- Remote inspection/execution/repair semantics are deliberately outside v0.2; version and specify them before implementation. diff --git a/LICENSE b/LICENSE index 077633a..f09ffe1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,10 +1,147 @@ Mozilla Public License Version 2.0 ================================== -This repository is licensed under the Mozilla Public License 2.0. -The canonical text is: +1. Definitions +-------------- -https://www.mozilla.org/MPL/2.0/ +1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. -This matches The-Interdependency/skill-lib. Changes to MPL-covered files -must be published under MPL-2.0. +1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” means Covered Software of a particular Contributor. + +1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. + +1.5. “Incompatible With Secondary Licenses” means +(a) that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or +(b) that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. + +1.6. “Executable Form” means any form of the work other than Source Code Form. + +1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. + +1.8. “License” means this document. + +1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. + +1.10. “Modifications” means any of the following: +(a) any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or +(b) any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. + +1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants +Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: +(a) under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and +(b) under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. + +2.2. Effective Date +The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. + +2.3. Limitations on Grant Scope +The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: +(a) for any code that a Contributor has removed from Covered Software; or +(b) for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or +(c) under Patent Claims infringed by Covered Software in the absence of its Contributions. + +This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). + +2.4. Subsequent Licenses +No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). + +2.5. Representation +Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use +This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form +All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form +If You distribute Covered Software in Executable Form then: +(a) such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and +(b) You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work +You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). + +3.4. Notices +You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms +You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. + +6. Disclaimer of Warranty +------------------------- + +Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. + +7. Limitation of Liability +-------------------------- + +Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions +Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. + +10.2. Effect of New Versions +You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. + +10.3. Modified Versions +If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses +If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. + +Exhibit B - “Incompatible With Secondary Licenses” Notice +--------------------------------------------------------- + +This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. diff --git a/README.md b/README.md index 4ff731a..a1eccb7 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,17 @@ Public distribution of [skill-lib](https://github.com/The-Interdependency/skill-lib). -Clone this repo when you want a command that inspects a repository and writes findings. The full catalog, org doctrine, and unfinished skills stay in skill-lib. This repo is the subset a stranger can run. +Clone this repo when you want a command that inspects a local repository and writes findings. The full catalog, org doctrine, and unfinished skills stay in skill-lib. This repo is the subset a stranger can run. ## Status -The inspect CLI ships. Clone, run, get findings. +The inspect CLI works on `main`; the `v0.2.0` release tag is not published yet. | Claim | State | |---|---| | Canon | `The-Interdependency/skill-lib` | | This repo | distribution + public CLI + fixtures | -| Clone / run / findings | **shipped** — `v0.2` inspect; see `HANDOFF.md` | +| Clone / run / findings | **implemented on main** — release pending | | VM populate | `HANDOFF.vm.md` | | Source pin | `SOURCE.md` | @@ -29,55 +29,46 @@ python -m unittest discover -s tests python -m pubskill_lib.audit examples/neglected-repo --out /tmp/findings.json ``` -Those commands are the definition of done for the first utility tag (`v0.2.0`). They run on a clean clone. +Those commands are the definition of done for the first utility tag (`v0.2.0`). They run in GitHub CI from a clean checkout; publish the tag only after the release gate is explicitly completed. -## What this will do +## Inspect CLI -Inspect one repository path or URL at a named commit and write: +`v0.2` inspects one **local repository path** without executing the target repository: -- identity (remote, commit, dirty state, declared instructions) -- claimed gates vs files that exist -- obvious dependency and docs drift -- findings classified as `defect`, `environment`, `external`, `policy`, or `hmmm` +```bash +python -m pubskill_lib.audit PATH --out findings.json +``` -`--run` is opt-in. `verified` is stamped only on a finding whose gate was re-run. +It writes: -## What this will not do +- identity when the target contains `.git` (remote, commit, dirty state) +- README links to missing local files +- obvious test-workflow no-ops +- Python `pyproject.toml` console scripts whose modules are missing +- findings classified as `defect`, `environment`, `external`, `policy`, or `hmmm` -- audit the whole internet -- execute private CI secrets by default -- rewrite a repo unless `--fix-one` is explicit and bounded -- carry The Interdependent Way, UCNS, or org liturgy on the first screen +The inspector does **not** yet clone URLs, select remote commits, execute target tests, or repair the target. Those are later capabilities and must not be inferred from the schema. ## Repository examiner (BYOK) -The inspect CLI is the first consumer of a repository evidence engine. A -documentation generator builds on that same evidence substrate: +A separate documentation examiner builds on the repository evidence substrate: ```bash -python -m pubskill_lib.examine --repo /path/to/repo --json # dry run -python -m pubskill_lib.examine --repo /path/to/repo --apply --narrate # write + assemble +python -m pubskill_lib.examine --repo /path/to/repo --json +python -m pubskill_lib.examine --repo /path/to/repo --apply --narrate ``` -With `--apply`, the examiner inventories actual code, writes a descriptive -`NARRATIVE` msdmd block into each supported source file (never a `CONTRACT`, -`CHECK`, `CAPABILITY`, or other normative declaration), maintains -shebang-first RATIOS placement, and assembles `docs/examiner/EXAMINER.md` -from the discovered module graph. Narratives are evidence-bound to the source -hash that produced them; changed source without a re-narrate is marked stale. -The tool never leaves the repository boundary it was pointed at. +With `--apply`, the examiner inventories actual code, writes descriptive `NARRATIVE` msdmd blocks into supported source files, maintains source-boundary RATIOS placement, and assembles `docs/examiner/EXAMINER.md`. Narratives are evidence-bound to the source hash that produced them; changed source without a re-narrate is marked stale. The tool never writes outside the repository boundary it was pointed at. -BYOK credentials come from the environment or a `.env` file and are never -printed: +BYOK credentials may come from the process environment or a `.env` file. `.env` files are ignored by this repository. To prevent a target repository from redirecting an operator credential, provider base-URL overrides are accepted only from the process environment, not from `.env`: ```text -OPENAI_API_KEY / OPENAI_BASE_URL / OPENAI_MODEL -ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL / ANTHROPIC_MODEL +OPENAI_API_KEY / OPENAI_MODEL +ANTHROPIC_API_KEY / ANTHROPIC_MODEL +OPENAI_BASE_URL / ANTHROPIC_BASE_URL # process environment only ``` -Multiple providers are attempted sequentially (fallback). Languages without -a shipped ratio computer keep `hmmm` values; unsupported languages are -skipped and reported as `hmmm`, never guessed. +Multiple configured providers are attempted sequentially as fallback. Unsupported or not-faithfully-computable metrics remain `hmmm`; they are not guessed. ## License @@ -85,4 +76,4 @@ MPL-2.0, same as skill-lib. Changes to MPL-covered files must be published. ## Canon -Do not add skills here first. Add them in skill-lib, mark them `runnable`, pin the SHA in `SOURCE.md`, then propagate. +Do not add skills here first. Add them in skill-lib, mark them appropriately, pin the SHA in `SOURCE.md`, then propagate the public slice. diff --git a/SOURCE.md b/SOURCE.md index 3c88a77..994533e 100644 --- a/SOURCE.md +++ b/SOURCE.md @@ -4,10 +4,10 @@ Canon: https://github.com/The-Interdependency/skill-lib | Field | Value | |---|---| -| Pinned SHA | `be72da66a112d0632fd25480c1f51b6e69db4976` | -| Pinned date | 2026-09-04 | -| Pin meaning | last observed skill-lib `main` when pubskill-lib was created | -| Runnable subset | `msdmd` (runnable), `repo-audit-repair` (contract) — set in skill-lib `skills.json` after the pin | +| Pinned SHA | `c14ee9d500579a4b5d6821f62c9d82ca96e73608` | +| Pinned date | 2026-09-06 | +| Pin meaning | exact canonical source used for the vendored public skill slice | +| Runnable subset | `msdmd` (runnable), `repo-audit-repair` (contract) | Update this file in the same commit that propagates vendored skills. diff --git a/examples/neglected-repo/expected-findings.json b/examples/neglected-repo/expected-findings.json index 0a4ca74..223546c 100644 --- a/examples/neglected-repo/expected-findings.json +++ b/examples/neglected-repo/expected-findings.json @@ -1,19 +1,14 @@ { "schema_version": 1, "tool": "pubskill-lib", - "source_pin": "be72da66a112d0632fd25480c1f51b6e69db4976", + "source_pin": "c14ee9d500579a4b5d6821f62c9d82ca96e73608", "target": { "path": "examples/neglected-repo", "commit": "hmmm", "remote": "hmmm", "dirty": false }, - "surfaces": [ - "identity", - "docs", - "ci", - "deps" - ], + "surfaces": ["identity", "docs", "ci", "deps"], "findings": [ { "id": "F001", @@ -43,7 +38,5 @@ "verified": false } ], - "hmmm": [ - "target has no .git directory; identity unresolved" - ] + "hmmm": ["target has no .git directory; identity unresolved"] } diff --git a/src/pubskill_lib.egg-info/PKG-INFO b/src/pubskill_lib.egg-info/PKG-INFO deleted file mode 100644 index 4700b39..0000000 --- a/src/pubskill_lib.egg-info/PKG-INFO +++ /dev/null @@ -1,68 +0,0 @@ -Metadata-Version: 2.4 -Name: pubskill-lib -Version: 0.2.0 -Summary: Public inspect CLI for evidence-led repository findings -License: MPL-2.0 -Requires-Python: >=3.11 -Description-Content-Type: text/markdown -License-File: LICENSE -Dynamic: license-file - -# pubskill-lib - -Public distribution of [skill-lib](https://github.com/The-Interdependency/skill-lib). - -Clone this repo when you want a command that inspects a repository and writes findings. The full catalog, org doctrine, and unfinished skills stay in skill-lib. This repo is the subset a stranger can run. - -## Status - -The repository exists. The runnable utility is not in the tree yet. - -| Claim | State | -|---|---| -| Canon | `The-Interdependency/skill-lib` | -| This repo | distribution + public CLI + fixtures | -| Clone / run / findings | **not shipped** — see `HANDOFF.md` | -| VM populate | `HANDOFF.vm.md` | -| Source pin | `SOURCE.md` | - -If a command is not in this README, it is not a public promise. - -## Intended quickstart (target, not current) - -```bash -git clone https://github.com/The-Interdependency/pubskill-lib -cd pubskill-lib -python -m venv .venv && source .venv/bin/activate -python -m pip install -e . -python -m unittest discover -s tests -python -m pubskill_lib.audit examples/neglected-repo --out /tmp/findings.json -``` - -Those commands are the definition of done for the first utility tag (`v0.2.0`). Until they work, do not treat this README as a product page. - -## What this will do - -Inspect one repository path or URL at a named commit and write: - -- identity (remote, commit, dirty state, declared instructions) -- claimed gates vs files that exist -- obvious dependency and docs drift -- findings classified as `defect`, `environment`, `external`, `policy`, or `hmmm` - -`--run` is opt-in. `verified` is stamped only on a finding whose gate was re-run. - -## What this will not do - -- audit the whole internet -- execute private CI secrets by default -- rewrite a repo unless `--fix-one` is explicit and bounded -- carry The Interdependent Way, UCNS, or org liturgy on the first screen - -## License - -MPL-2.0, same as skill-lib. Changes to MPL-covered files must be published. - -## Canon - -Do not add skills here first. Add them in skill-lib, mark them `runnable`, pin the SHA in `SOURCE.md`, then propagate. diff --git a/src/pubskill_lib.egg-info/SOURCES.txt b/src/pubskill_lib.egg-info/SOURCES.txt deleted file mode 100644 index 9f36f7f..0000000 --- a/src/pubskill_lib.egg-info/SOURCES.txt +++ /dev/null @@ -1,13 +0,0 @@ -LICENSE -README.md -pyproject.toml -src/pubskill_lib/__init__.py -src/pubskill_lib/audit.py -src/pubskill_lib/schema.py -src/pubskill_lib.egg-info/PKG-INFO -src/pubskill_lib.egg-info/SOURCES.txt -src/pubskill_lib.egg-info/dependency_links.txt -src/pubskill_lib.egg-info/entry_points.txt -src/pubskill_lib.egg-info/top_level.txt -tests/test_audit_fixture.py -tests/test_schema.py \ No newline at end of file diff --git a/src/pubskill_lib.egg-info/dependency_links.txt b/src/pubskill_lib.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/src/pubskill_lib.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/pubskill_lib.egg-info/entry_points.txt b/src/pubskill_lib.egg-info/entry_points.txt deleted file mode 100644 index 4c26d9f..0000000 --- a/src/pubskill_lib.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -pubskill-audit = pubskill_lib.audit:main diff --git a/src/pubskill_lib.egg-info/top_level.txt b/src/pubskill_lib.egg-info/top_level.txt deleted file mode 100644 index cc9b8de..0000000 --- a/src/pubskill_lib.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -pubskill_lib diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 6a1c22d..a8eba6f 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -3,9 +3,8 @@ Usage: python -m pubskill_lib.audit PATH --out findings.json -v0.2 inspect only: read declared files, record identity, flag evidenced -repository defects. Never install target deps, never run target tests. -Exit 0 when the tool ran; exit 3 on tool/schema failures. +v0.2 inspect only: read declared files, record identity, and flag evidenced +repository defects. It never installs target dependencies or runs target tests. """ import argparse @@ -27,6 +26,9 @@ ECHO_OR_NOOP_PATTERN = re.compile(r"\b(echo|true|exit\s+0|printf)\b", re.IGNORECASE) MARKDOWN_LINK_PATTERN = re.compile(r"\[[^\]]*\]\(([^)]+)\)") PIN_PATTERN = re.compile(r"`([0-9a-f]{40})`") +LOCAL_SCRIPT_PATTERN = re.compile( + r"(?:^|(?:&&|;|\|)\s*)(?:node|python(?:3)?|bash|sh)\s+([^\s;&|]+)" +) class _Sink: @@ -65,12 +67,13 @@ def _git_identity(target): def run(args): try: - return subprocess.run( + result = subprocess.run( ["git", "-C", str(target), *args], capture_output=True, text=True, timeout=10, - ).stdout.strip() + ) + return result.stdout.strip() if result.returncode == 0 else None except (OSError, subprocess.SubprocessError): return None @@ -95,12 +98,13 @@ def _check_readme_links(target, sink): if local.startswith("/"): continue resolved = (readme.parent / local).resolve() + try: + resolved.relative_to(target.resolve()) + except ValueError: + sink.add("docs", f"README link escapes repository: {dest}", f"{name}:{lineno}") + continue if not resolved.exists(): - sink.add( - "docs", - f"README links to {dest}", - f"{name}:{lineno}", - ) + sink.add("docs", f"README links to {dest}", f"{name}:{lineno}") def _check_ci_workflows(target, sink): @@ -135,30 +139,60 @@ def _module_exists(target, module): return any(candidate.exists() for candidate in candidates) -def _check_declared_scripts(target, sink): +def _check_pyproject_scripts(target, sink): pyproject = target / "pyproject.toml" text = _read_text(pyproject) if text is None: return try: import tomllib - except ImportError: # pragma: no cover - requires Python 3.11+ - return - try: data = tomllib.loads(text) - except Exception: + except (ImportError, ValueError): return scripts = (data.get("project") or {}).get("scripts") or {} for name in sorted(scripts): entry = str(scripts[name]) module = entry.split(":", 1)[0].strip() - if not module or _module_exists(target, module): + if module and not _module_exists(target, module): + sink.add( + "deps", + f"declared script {name} points to missing module {module}", + "pyproject.toml [project.scripts]", + ) + + +def _check_package_scripts(target, sink): + package = target / "package.json" + text = _read_text(package) + if text is None: + return + try: + data = json.loads(text) + except json.JSONDecodeError: + sink.add("deps", "package.json is not valid JSON", "package.json") + return + scripts = data.get("scripts") or {} + if not isinstance(scripts, dict): + return + for name, command in sorted(scripts.items()): + if not isinstance(command, str): continue - sink.add( - "deps", - f"declared script {name} points to missing module {module}", - "pyproject.toml [project.scripts]", - ) + for match in LOCAL_SCRIPT_PATTERN.finditer(command): + raw_path = match.group(1).strip('"\'') + if raw_path.startswith(("-", "/")) or "://" in raw_path: + continue + local = (target / raw_path).resolve() + try: + local.relative_to(target.resolve()) + except ValueError: + sink.add("deps", f"package script {name} escapes repository via {raw_path}", "package.json [scripts]") + continue + if not local.exists(): + sink.add( + "deps", + f"package script {name} points to missing local file {raw_path}", + "package.json [scripts]", + ) def _read_source_pin(): @@ -169,7 +203,6 @@ def _read_source_pin(): def audit_path(target_path, source_pin=None): - """Inspect one repository path and return a schema-valid document.""" target = Path(target_path) source_pin = source_pin or _read_source_pin() document = new_document(source_pin, target_path) @@ -191,9 +224,14 @@ def audit_path(target_path, source_pin=None): surfaces.append("ci") _check_ci_workflows(target, sink) - if (target / "pyproject.toml").exists() or (target / "package.json").exists(): + has_pyproject = (target / "pyproject.toml").exists() + has_package = (target / "package.json").exists() + if has_pyproject or has_package: surfaces.append("deps") - _check_declared_scripts(target, sink) + if has_pyproject: + _check_pyproject_scripts(target, sink) + if has_package: + _check_package_scripts(target, sink) document["surfaces"] = surfaces document["findings"] = sink.finalize() @@ -203,7 +241,7 @@ def audit_path(target_path, source_pin=None): def main(argv=None): parser = argparse.ArgumentParser(prog="python -m pubskill_lib.audit") - parser.add_argument("path", help="repository path to inspect") + parser.add_argument("path", help="local repository path to inspect") parser.add_argument("--out", required=True, help="findings.json output path") args = parser.parse_args(argv) @@ -214,7 +252,7 @@ def main(argv=None): try: document = audit_path(target) - except Exception as exc: # tool/schema failure, never the target's fault + except Exception as exc: print(f"pubskill_lib.audit: tool failure: {exc}", file=sys.stderr) return 3 diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index b6a4f2c..2031a10 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -1,34 +1,36 @@ """Evidence engine: inventory actual code before describing it. -This layer never infers behavior from names, layout, or convention. It reads -files and records what is actually there: language marker, shebang, existing -msdmd blocks, existing RATIOS lines, content hash, size, and executable bit. +Language comment markers are loaded from the vendored canonical msdmd parser; +this module does not maintain a second registry. """ from __future__ import annotations import hashlib +import importlib.util import re from dataclasses import dataclass, field +from functools import lru_cache from pathlib import Path from . import boundary -# extension -> comment marker (same table as canon msdmd) -MARKERS: dict[str, str] = { - ".py": "#", ".rb": "#", ".ex": "#", ".exs": "#", ".sh": "#", - ".ts": "//", ".tsx": "//", ".js": "//", ".jsx": "//", ".mjs": "//", - ".rs": "//", ".go": "//", ".java": "//", ".c": "//", ".cpp": "//", - ".cc": "//", ".h": "//", ".hpp": "//", ".swift": "//", ".kt": "//", - ".sql": "--", ".lua": "--", ".hs": "--", -} - SHEBANG_RE = re.compile(r"^#!.*$") -_RATIOS_LINE_RE = re.compile(r"^(?:#|//|--)\s*ratios:\s*(.+?)\s*$") -_NARRATIVE_FENCE_RE = re.compile( - r"^(?:#|//|--) === NARRATIVE ===\s*$.*?^(?:#|//|--) === END NARRATIVE ===\s*$", - re.MULTILINE | re.DOTALL, -) +_RATIOS_LINE_RE = re.compile(r"^(?:#|//|--|%|;|!|'|\*>)\s*ratios:\s*(.+?)\s*$") + + +@lru_cache(maxsize=1) +def _comment_markers() -> dict[str, str]: + """Load COMMENT_MARKERS from the pinned repo-local msdmd parser.""" + repo = Path(__file__).resolve().parents[2] + parser_path = repo / ".agents" / "skills" / "msdmd" / "parsers" / "universal.py" + spec = importlib.util.spec_from_file_location("pubskill_lib._vendored_msdmd", parser_path) + if spec is None or spec.loader is None: + return {} + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + markers = getattr(module, "COMMENT_MARKERS", {}) + return dict(markers) if isinstance(markers, dict) else {} @dataclass @@ -48,75 +50,74 @@ class FileEvidence: def _block_name_re(marker: str) -> re.Pattern[str]: m = re.escape(marker) - return re.compile(rf"^{m} === ([A-Z_]+) ===\s*$(?P.*?)^{m} === END \1 ===\s*$", re.MULTILINE | re.DOTALL) + return re.compile( + rf"^{m} === ([A-Z_]+) ===\s*$(?P.*?)^{m} === END \1 ===\s*$", + re.MULTILINE | re.DOTALL, + ) def _parse_block_entries(marker: str, body: str) -> list[dict]: m = re.escape(marker) id_re = re.compile(rf"^\s*{m}\s*id:\s*(?P\S+)\s*$") field_re = re.compile(rf"^\s*{m}\s+(?P[a-z_]+):\s*(?P.+?)\s*$") - entries: list[dict] = [] + entries: list[dict[str, str]] = [] current: dict[str, str] | None = None for line in body.splitlines(): line = line.rstrip() - mid = id_re.match(line) - if mid: + match_id = id_re.match(line) + if match_id: if current is not None: entries.append(current) - current = {"id": mid.group("id")} + current = {"id": match_id.group("id")} continue if current is None: continue - mf = field_re.match(line) - if mf: - current[mf.group("key")] = mf.group("val") + match_field = field_re.match(line) + if match_field: + current[match_field.group("key")] = match_field.group("val") if current is not None: entries.append(current) return entries def read_evidence(root: Path, path: Path) -> FileEvidence: - """Read one file into evidence. Never raises for unsupported files.""" root = Path(root).resolve() rel = str(path.relative_to(root)) - marker = MARKERS.get(path.suffix.lower()) + marker = _comment_markers().get(path.suffix.lower()) language = path.suffix.lower().lstrip(".") or "unknown" - evidence = FileEvidence(path=rel, language=language, marker=marker) + item = FileEvidence(path=rel, language=language, marker=marker) try: text = path.read_text(encoding="utf-8", errors="replace") except OSError: - evidence.hmmm.append("unreadable file") - return evidence - evidence.sha256 = hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest() - evidence.size = len(text.encode("utf-8", errors="replace")) + item.hmmm.append("unreadable file") + return item + + encoded = text.encode("utf-8", errors="replace") + item.sha256 = hashlib.sha256(encoded).hexdigest() + item.size = len(encoded) try: - evidence.executable = bool(path.stat().st_mode & 0o111) + item.executable = bool(path.stat().st_mode & 0o111) except OSError: pass first_line = text.splitlines()[0].rstrip() if text.splitlines() else "" if SHEBANG_RE.match(first_line): - evidence.shebang = first_line + item.shebang = first_line if marker is not None: for raw in text.splitlines(): if _RATIOS_LINE_RE.match(raw.rstrip()): - evidence.ratios_lines.append(raw.rstrip()) - block_re = _block_name_re(marker) - for match in block_re.finditer(text): + item.ratios_lines.append(raw.rstrip()) + for match in _block_name_re(marker).finditer(text): name = match.group(1) entries = _parse_block_entries(marker, match.group("body")) - evidence.msdmd_blocks.setdefault(name, []).extend(entries) - evidence.narrative_entries = evidence.msdmd_blocks.get("NARRATIVE", []) + item.msdmd_blocks.setdefault(name, []).extend(entries) + item.narrative_entries = item.msdmd_blocks.get("NARRATIVE", []) else: - evidence.hmmm.append(f"unsupported language for msdmd: .{language}") - return evidence + item.hmmm.append(f"unsupported language for msdmd: .{language}") + return item def inventory(root: Path) -> list[FileEvidence]: - """Inventory every regular file inside the boundary.""" root = boundary.assert_inside(root, root) - out: list[FileEvidence] = [] - for path in boundary.iter_files(root): - out.append(read_evidence(root, path)) - return out + return [read_evidence(root, path) for path in boundary.iter_files(root)] diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index ddbd40b..82c8d4c 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -4,11 +4,8 @@ python -m pubskill_lib.examine [--repo PATH] [--apply] [--narrate] [--out DIR] [--env FILE] -Layers are separable: evidence (inventory), model reasoning (narrate), -source mutation (msdmd writer + RATIOS), documentation assembly, and -provider access each live in their own module and can be used directly. - -Default is a dry run: report what would change without writing anything. +Default is a dry run. Evidence, model reasoning, source mutation, +documentation assembly, and provider access remain separate layers. """ from __future__ import annotations @@ -16,7 +13,6 @@ import argparse import json import sys -from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path @@ -50,7 +46,12 @@ def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: } -def _apply(root: Path, evidence_list: list[evidence.FileEvidence], provider_list: list[providers.Provider], narrate: bool) -> tuple[dict, dict]: +def _apply( + root: Path, + evidence_list: list[evidence.FileEvidence], + provider_list: list[providers.Provider], + narrate: bool, +) -> tuple[dict, dict]: narratives: dict[str, dict[str, str]] = {} changed: list[str] = [] now = datetime.now(timezone.utc).isoformat() @@ -80,7 +81,9 @@ def _apply(root: Path, evidence_list: list[evidence.FileEvidence], provider_list entry = narratives.get(ev.path) if entry: - new_text, block_changed = msdmd_writer.upsert_narrative(new_text, ev.marker, entry) + new_text, block_changed = msdmd_writer.upsert_narrative( + new_text, ev.marker, entry, path + ) if block_changed: changed.append(f"{ev.path}:narrative") @@ -93,7 +96,7 @@ def _apply(root: Path, evidence_list: list[evidence.FileEvidence], provider_list def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="python -m pubskill_lib.examine") parser.add_argument("--repo", help="repository path (default: current directory)") - parser.add_argument("--apply", action="store_true", help="write msdmd + RATIOS + docs (default: dry run)") + parser.add_argument("--apply", action="store_true", help="write msdmd + RATIOS + docs") parser.add_argument("--narrate", action="store_true", help="call BYOK providers for narratives") parser.add_argument("--env", default=".env", help=".env file for BYOK credentials") parser.add_argument("--out", default="docs/examiner", help="documentation output directory") diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index b29480d..1c76ea4 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -1,16 +1,17 @@ """msdmd NARRATIVE writer: generated explanation is descriptive evidence. -The NARRATIVE block is a normal msdmd fenced block, never a CONTRACT, CHECK, -CAPABILITY, OWNERS, DOCS, or other normative declaration. It carries the -content hash that produced it so stale narrative can be detected. +NARRATIVE placement shares the language opening-boundary rules used by the +RATIOS engine so metadata never displaces an interpreter or protected source +prologue. """ from __future__ import annotations -import hashlib import re from pathlib import Path +from . import ratios + NARRATIVE_BLOCK = "NARRATIVE" @@ -31,26 +32,26 @@ def _block_lines(marker: str, entry: dict[str, str]) -> list[str]: def narrative_id(sha256: str) -> str: - """Stable, refactor-safe entry id derived from the evidence hash.""" return f"examiner_{sha256[:16]}" -def upsert_narrative(text: str, marker: str, entry: dict[str, str]) -> tuple[str, bool]: - """Replace any existing NARRATIVE blocks with ``entry``, keeping the - shebang first and the ratios bookends where they were.""" - fence = _fence_re(marker) - body = fence.sub("", text).rstrip("\n") +def upsert_narrative( + text: str, + marker: str, + entry: dict[str, str], + path: Path | None = None, +) -> tuple[str, bool]: + """Replace NARRATIVE blocks without crossing the protected opening boundary.""" + body = _fence_re(marker).sub("", text).rstrip("\n") block = "\n".join(_block_lines(marker, entry)) - lines = body.splitlines() + + adapter = ratios.RatiosEngine().adapter_for(path) if path is not None else None + insert_at = ratios.opening_index(lines, adapter) ratios_prefix = f"{marker} ratios:" - insert_at = 0 - if lines and lines[0].startswith("#!"): - insert_at = 1 - if len(lines) > 1 and lines[1].lstrip().startswith(ratios_prefix): - insert_at = 2 - elif lines and lines[0].lstrip().startswith(ratios_prefix): - insert_at = 1 + if insert_at < len(lines) and lines[insert_at].lstrip().startswith(ratios_prefix): + insert_at += 1 + lines.insert(insert_at, block) new_text = "\n".join(lines) + "\n" return new_text, new_text != text diff --git a/src/pubskill_lib/narrative.py b/src/pubskill_lib/narrative.py index 243c406..a266194 100644 --- a/src/pubskill_lib/narrative.py +++ b/src/pubskill_lib/narrative.py @@ -1,8 +1,7 @@ """Model reasoning layer: evidence-bound narrative generation. -A narrative is descriptive evidence about one file, bound to the exact -content hash that produced it. If the file changes and the narrative does -not, the narrative is stale and the assembler flags it. +A narrative is descriptive evidence about one file, bound to the exact content +hash that produced it. Successful generations also record provider and model. """ from __future__ import annotations @@ -46,7 +45,6 @@ def narrate_file( provider_list: list[providers.Provider], now: str, ) -> Narrative: - """Return an evidence-bound narrative entry for one file.""" entry = { "id": narrative_id(ev.sha256), "summary": "", @@ -68,12 +66,13 @@ def narrate_file( entry["provider"] = "none" else: try: - summary, name = providers.chat_with_fallback( + summary, name, model = providers.chat_with_fallback( provider_list, SYSTEM_PROMPT, _user_prompt(ev, text) ) entry["summary"] = " ".join(summary.split()) entry["provider"] = name - except Exception as exc: # noqa: BLE001 - provider failure is hmmm + entry["model"] = model or "hmmm" + except Exception as exc: # provider failure remains visible as hmmm hmmm.append(f"model reasoning failed: {type(exc).__name__}") if hmmm: @@ -83,5 +82,4 @@ def narrate_file( def is_stale(entry: dict[str, str], current_sha256: str) -> bool: - """A narrative is stale when its evidence hash no longer matches.""" return entry.get("evidence_sha256") != current_sha256 diff --git a/src/pubskill_lib/providers.py b/src/pubskill_lib/providers.py index 13eab5e..ca021e2 100644 --- a/src/pubskill_lib/providers.py +++ b/src/pubskill_lib/providers.py @@ -1,13 +1,8 @@ -"""BYOK provider access. Credentials come from environment/.env and are -never printed, logged, or returned by this layer. +"""BYOK provider access. -Supported providers (configurable through environment): - - OPENAI_API_KEY, OPENAI_BASE_URL (default api.openai.com/v1), OPENAI_MODEL - ANTHROPIC_API_KEY, ANTHROPIC_BASE_URL (default api.anthropic.com), ANTHROPIC_MODEL - -Multiple providers are attempted sequentially by default; callers may also -drive them concurrently across files. +Credentials may come from the process environment or a local .env file. Base +URL overrides are process-environment-only so a target repository cannot pair +an operator's ambient API key with a repository-controlled endpoint. """ from __future__ import annotations @@ -19,6 +14,7 @@ DEFAULT_OPENAI_BASE = "https://api.openai.com/v1" DEFAULT_ANTHROPIC_BASE = "https://api.anthropic.com" +_BASE_URL_KEYS = {"OPENAI_BASE_URL", "ANTHROPIC_BASE_URL"} def load_dotenv(path: str | Path = ".env") -> dict[str, str]: @@ -41,8 +37,10 @@ def load_dotenv(path: str | Path = ".env") -> dict[str, str]: def env_with_dotenv(path: str | Path = ".env") -> dict[str, str]: - """Merged os.environ plus .env values (os.environ wins).""" + """Merge .env with os.environ; base URL overrides come only from os.environ.""" merged = dict(load_dotenv(path)) + for key in _BASE_URL_KEYS: + merged.pop(key, None) merged.update(os.environ) return merged @@ -65,13 +63,13 @@ def __init__(self, name: str, env: dict[str, str]): self.key = env.get(self.key_env(), "") self.model = env.get(self.model_env(), self.default_model()) - def key_env(self) -> str: # pragma: no cover - overridden + def key_env(self) -> str: raise NotImplementedError - def model_env(self) -> str: # pragma: no cover - overridden + def model_env(self) -> str: raise NotImplementedError - def default_model(self) -> str: # pragma: no cover - overridden + def default_model(self) -> str: raise NotImplementedError def configured(self) -> bool: @@ -80,7 +78,7 @@ def configured(self) -> bool: def describe(self) -> str: return f"{self.name} model={self.model or 'hmmm'} key={_mask(self.key) if self.key else 'absent'}" - def chat(self, system: str, user: str) -> str: # pragma: no cover - overridden + def chat(self, system: str, user: str) -> str: raise NotImplementedError @@ -100,10 +98,7 @@ def default_model(self) -> str: return "gpt-4o-mini" def chat(self, system: str, user: str) -> str: - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.key}", - } + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.key}"} payload = { "model": self.model, "messages": [ @@ -148,17 +143,19 @@ def chat(self, system: str, user: str) -> str: def configured_providers(env: dict[str, str]) -> list[Provider]: - """Return providers with credentials, in stable order.""" + """Return configured providers in stable fallback order.""" providers = [OpenAIProvider(env), AnthropicProvider(env)] - return [p for p in providers if p.configured()] + return [provider for provider in providers if provider.configured()] -def chat_with_fallback(providers: list[Provider], system: str, user: str) -> tuple[str, str]: - """Try providers sequentially. Returns (text, provider_name) or raises.""" +def chat_with_fallback( + providers: list[Provider], system: str, user: str +) -> tuple[str, str, str]: + """Try providers sequentially. Return (text, provider_name, model).""" errors: list[str] = [] for provider in providers: try: - return provider.chat(system, user), provider.name - except Exception as exc: # noqa: BLE001 - boundary to hmmm, never to crash + return provider.chat(system, user), provider.name, provider.model + except Exception as exc: # boundary failure remains visible without leaking secrets errors.append(f"{provider.name}: {type(exc).__name__}") raise RuntimeError("; ".join(errors) or "no providers configured") diff --git a/tests/test_repairs.py b/tests/test_repairs.py new file mode 100644 index 0000000..d26a46b --- /dev/null +++ b/tests/test_repairs.py @@ -0,0 +1,117 @@ +"""Regression tests for audit findings repaired on 2026-09-10.""" + +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from pubskill_lib import audit, evidence, msdmd_writer, narrative, providers + + +class CredentialBoundaryTests(unittest.TestCase): + def test_dotenv_cannot_override_provider_base_url(self): + with tempfile.TemporaryDirectory() as tmp: + env_file = Path(tmp) / ".env" + env_file.write_text( + "OPENAI_API_KEY=repo-key\nOPENAI_BASE_URL=https://attacker.invalid/v1\n", + encoding="utf-8", + ) + with patch.dict(os.environ, {"OPENAI_API_KEY": "operator-key"}, clear=True): + merged = providers.env_with_dotenv(env_file) + self.assertEqual("operator-key", merged["OPENAI_API_KEY"]) + self.assertNotIn("OPENAI_BASE_URL", merged) + + def test_process_environment_may_set_base_url(self): + with tempfile.TemporaryDirectory() as tmp: + env_file = Path(tmp) / ".env" + env_file.write_text("OPENAI_BASE_URL=https://attacker.invalid/v1\n", encoding="utf-8") + with patch.dict( + os.environ, + { + "OPENAI_API_KEY": "operator-key", + "OPENAI_BASE_URL": "https://operator.example/v1", + }, + clear=True, + ): + merged = providers.env_with_dotenv(env_file) + self.assertEqual("https://operator.example/v1", merged["OPENAI_BASE_URL"]) + + +class NarrativeBoundaryTests(unittest.TestCase): + def test_narrative_preserves_python_shebang_and_encoding_header(self): + text = ( + "#!/usr/bin/env python3\n" + "# -*- coding: latin-1 -*-\n" + "# ratios: loc_comments=1:1 imports_exports=0:0 calls_definitions=1:0\n" + "print('hi')\n" + "# ratios: loc_comments=1:1 imports_exports=0:0 calls_definitions=1:0\n" + ) + entry = { + "id": "examiner_abc", + "summary": "Prints hi.", + "evidence_sha256": "abc", + "model": "none", + "provider": "none", + "generated_at": "now", + "stale": "false", + } + new, changed = msdmd_writer.upsert_narrative(text, "#", entry, Path("tool.py")) + lines = new.splitlines() + self.assertTrue(changed) + self.assertTrue(lines[0].startswith("#!")) + self.assertIn("coding:", lines[1]) + self.assertTrue(lines[2].startswith("# ratios:")) + self.assertEqual("# === NARRATIVE ===", lines[3]) + + def test_successful_narrative_records_model(self): + class FakeProvider: + name = "fake" + model = "model-1" + + def chat(self, system, user): + return "Does one thing." + + ev = evidence.FileEvidence(path="x.py", language="py", marker="#", sha256="abc") + result = narrative.narrate_file(ev, "print('x')\n", [FakeProvider()], "now") + self.assertEqual("fake", result.entry["provider"]) + self.assertEqual("model-1", result.entry["model"]) + + +class CanonicalMarkerTests(unittest.TestCase): + def test_evidence_uses_vendored_msdmd_registry(self): + markers = evidence._comment_markers() + self.assertEqual("#", markers[".py"]) + self.assertEqual("//", markers[".ts"]) + self.assertIn(".ps1", markers) + + +class PackageScriptTests(unittest.TestCase): + def test_missing_local_package_script_is_a_defect(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "package.json").write_text( + json.dumps({"scripts": {"build": "node scripts/build.js"}}), + encoding="utf-8", + ) + document = audit.audit_path(root, "pin") + findings = [f for f in document["findings"] if f["surface"] == "deps"] + self.assertEqual(1, len(findings)) + self.assertIn("scripts/build.js", findings[0]["claim"]) + + def test_existing_local_package_script_passes(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "scripts").mkdir() + (root / "scripts" / "build.js").write_text("console.log('ok')\n", encoding="utf-8") + (root / "package.json").write_text( + json.dumps({"scripts": {"build": "node scripts/build.js"}}), + encoding="utf-8", + ) + document = audit.audit_path(root, "pin") + self.assertFalse([f for f in document["findings"] if f["surface"] == "deps"]) + + +if __name__ == "__main__": + unittest.main() From 3a2f415d839930703dc9b86c1c8d0e17dfa86707 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Thu, 10 Sep 2026 17:43:01 -0700 Subject: [PATCH 02/39] repair: align documentation and provenance guards --- src/pubskill_lib/assemble.py | 51 +++++++++++++++++++++++++----------- tests/test_provenance.py | 33 +++++++++++++++++++++++ 2 files changed, 69 insertions(+), 15 deletions(-) create mode 100644 tests/test_provenance.py diff --git a/src/pubskill_lib/assemble.py b/src/pubskill_lib/assemble.py index 7b82d73..3fe2e2a 100644 --- a/src/pubskill_lib/assemble.py +++ b/src/pubskill_lib/assemble.py @@ -1,16 +1,16 @@ -"""Documentation assembly from the discovered module graph. +"""Documentation assembly from discovered repository structure. -Structure follows the architecture actually discovered, not directory depth -mechanically mirrored: +The assembler organizes inventoried files by their actual source paths: volume -> one markdown document per repository - parts -> top-level directories that contain modules - chapters-> nested directories with at least one module - sections-> individual module files + parts -> top-level directories that contain inventoried files + chapters-> nested directories with at least one inventoried file + sections-> individual files lists -> narrative summaries and gap/hmmm roll-ups -Files that exist but produced no narrative appear as lists with their hmmm -state, so gaps remain visible instead of being invented. +This is a filesystem-derived documentation hierarchy, not a dependency graph. +Dependency discovery may inform a future renderer, but this module does not +claim graph semantics it does not consume. """ from __future__ import annotations @@ -48,22 +48,32 @@ def _heading(path: str) -> str: return path.replace("_", " ").replace("-", " ") -def build_structure(evidence_list: list[FileEvidence], narratives: dict[str, dict[str, str]]) -> dict[str, Part]: +def build_structure( + evidence_list: list[FileEvidence], narratives: dict[str, dict[str, str]] +) -> dict[str, Part]: parts: dict[str, Part] = {} for ev in evidence_list: entry = narratives.get(ev.path) summary = entry.get("summary", "") if entry else "" stale = bool(entry) and is_stale(entry, ev.sha256) - hmmm = [h for h in ev.hmmm] + hmmm = list(ev.hmmm) if ev.marker is not None and not entry: hmmm.append("no narrative generated") - section = Section(path=ev.path, heading=_heading(ev.path), summary=summary, stale=stale, hmmm=hmmm) + section = Section( + path=ev.path, + heading=_heading(ev.path), + summary=summary, + stale=stale, + hmmm=hmmm, + ) rel = Path(ev.path) top = rel.parts[0] if len(rel.parts) > 1 else "(root)" part = parts.setdefault(top, Part(title=_heading(top))) chapter_key = "/".join(rel.parts[1:-1]) or "(root)" - chapter = part.chapters.setdefault(chapter_key, Chapter(title=_heading(chapter_key))) + chapter = part.chapters.setdefault( + chapter_key, Chapter(title=_heading(chapter_key)) + ) chapter.sections.append(section) return parts @@ -85,7 +95,11 @@ def _render_section(section: Section) -> list[str]: return lines -def render_markdown(root: Path, evidence_list: list[FileEvidence], narratives: dict[str, dict[str, str]]) -> str: +def render_markdown( + root: Path, + evidence_list: list[FileEvidence], + narratives: dict[str, dict[str, str]], +) -> str: parts = build_structure(evidence_list, narratives) lines = [ "# Repository examination", @@ -112,11 +126,18 @@ def render_markdown(root: Path, evidence_list: list[FileEvidence], narratives: d return "\n".join(lines) -def assemble_docs(root: Path, evidence_list: list[FileEvidence], narratives: dict[str, dict[str, str]], out_dir: Path) -> Path: +def assemble_docs( + root: Path, + evidence_list: list[FileEvidence], + narratives: dict[str, dict[str, str]], + out_dir: Path, +) -> Path: """Write the assembled volume and return its path.""" root = boundary.assert_inside(root, root) out = boundary.assert_inside(root, out_dir) out.mkdir(parents=True, exist_ok=True) volume = out / "EXAMINER.md" - volume.write_text(render_markdown(root, evidence_list, narratives), encoding="utf-8") + volume.write_text( + render_markdown(root, evidence_list, narratives), encoding="utf-8" + ) return volume diff --git a/tests/test_provenance.py b/tests/test_provenance.py new file mode 100644 index 0000000..c941f15 --- /dev/null +++ b/tests/test_provenance.py @@ -0,0 +1,33 @@ +import re +import unittest +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[1] +PIN_RE = re.compile(r"`([0-9a-f]{40})`") + + +class PublicationProvenanceTests(unittest.TestCase): + def test_source_pin_matches_vendored_skill_manifest(self): + source = (REPO / "SOURCE.md").read_text(encoding="utf-8") + vendored = (REPO / ".agents" / "skills" / "README.md").read_text(encoding="utf-8") + source_pin = PIN_RE.search(source) + vendored_pin = PIN_RE.search(vendored) + self.assertIsNotNone(source_pin) + self.assertIsNotNone(vendored_pin) + self.assertEqual(source_pin.group(1), vendored_pin.group(1)) + + def test_local_secret_files_are_ignored(self): + ignore = (REPO / ".gitignore").read_text(encoding="utf-8").splitlines() + self.assertIn(".env", ignore) + self.assertIn(".env.*", ignore) + self.assertIn("*.egg-info/", ignore) + + def test_readme_does_not_claim_unpublished_v020_tag(self): + readme = (REPO / "README.md").read_text(encoding="utf-8") + self.assertIn("release pending", readme) + self.assertNotIn("**shipped** — `v0.2`", readme) + + +if __name__ == "__main__": + unittest.main() From a7976df733b428b93ba0db921980be23c23db57e Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Thu, 10 Sep 2026 17:45:26 -0700 Subject: [PATCH 03/39] repair: remove stale hmmm and keep legal surface unchanged --- HANDOFF.md | 5 +- LICENSE | 147 ++--------------------------------------------------- 2 files changed, 7 insertions(+), 145 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 01e8005..65ecc7e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -39,7 +39,7 @@ python -m unittest discover -s tests python -m pubskill_lib.audit examples/neglected-repo --out /tmp/out.json ``` -The first release tag is exactly `v0.2.0`. README may say the implementation is ready on `main` before that tag exists; it must not claim the release exists before the tag is published. +GitHub CI executes this gate on Python 3.11 and 3.12. The first release tag is exactly `v0.2.0`. README may say the implementation is ready on `main` before that tag exists; it must not claim the release exists before the tag is published. ## Generated metadata @@ -47,5 +47,4 @@ The first release tag is exactly `v0.2.0`. README may say the implementation is ## hmmm -- Python 3.11 is part of the declared `>=3.11` support range but the current GitHub CI lane is Python 3.12 only. Add a 3.11 CI lane before treating cross-version support as independently witnessed. -- Remote inspection/execution/repair semantics are deliberately outside v0.2; version and specify them before implementation. +- Remote inspection, execution, and repair semantics are deliberately outside v0.2; version and specify them before implementation. diff --git a/LICENSE b/LICENSE index f09ffe1..077633a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,147 +1,10 @@ Mozilla Public License Version 2.0 ================================== -1. Definitions --------------- +This repository is licensed under the Mozilla Public License 2.0. +The canonical text is: -1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. +https://www.mozilla.org/MPL/2.0/ -1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” means Covered Software of a particular Contributor. - -1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. - -1.5. “Incompatible With Secondary Licenses” means -(a) that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or -(b) that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. - -1.6. “Executable Form” means any form of the work other than Source Code Form. - -1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. - -1.8. “License” means this document. - -1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. - -1.10. “Modifications” means any of the following: -(a) any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or -(b) any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. - -1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. - -1.13. “Source Code Form” means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants -Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: -(a) under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and -(b) under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. - -2.2. Effective Date -The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. - -2.3. Limitations on Grant Scope -The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: -(a) for any code that a Contributor has removed from Covered Software; or -(b) for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or -(c) under Patent Claims infringed by Covered Software in the absence of its Contributions. - -This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). - -2.4. Subsequent Licenses -No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). - -2.5. Representation -Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use -This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form -All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form -If You distribute Covered Software in Executable Form then: -(a) such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and -(b) You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work -You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). - -3.4. Notices -You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms -You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. - -6. Disclaimer of Warranty -------------------------- - -Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. - -7. Limitation of Liability --------------------------- - -Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions -Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. - -10.2. Effect of New Versions -You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. - -10.3. Modified Versions -If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses -If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - -This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. - -Exhibit B - “Incompatible With Secondary Licenses” Notice ---------------------------------------------------------- - -This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. +This matches The-Interdependency/skill-lib. Changes to MPL-covered files +must be published under MPL-2.0. From fbc38422ddee2a28f2ba1e4dd4eccda3a6a69f93 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Thu, 10 Sep 2026 17:48:18 -0700 Subject: [PATCH 04/39] repair: preserve handoff scope and public claims --- HANDOFF.md | 179 ++++++++++++++++++++++++++++++++++++++++++++++++----- README.md | 7 ++- 2 files changed, 166 insertions(+), 20 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 65ecc7e..ae61d01 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,50 +1,195 @@ -# HANDOFF — pubskill-lib v0.2 closure +# HANDOFF — populate pubskill-lib with clone utility + +This file is the work order. Execute in order. Do not skip to features. + +If the host is a VM, read `HANDOFF.vm.md` first. VM default: steps A–F only. No push, no tag, no skill-lib writes. Owner repo: `The-Interdependency/pubskill-lib` Canon: `The-Interdependency/skill-lib` Current pin: see `SOURCE.md` Public claims: `README.md` -Agent contract: `AGENTS.md` +Agent contract: `AGENTS.md` +VM contract: `HANDOFF.vm.md` ## Goal -A stranger clones this repository, installs it, runs its tests, audits `examples/neglected-repo`, and gets the expected findings file. That is the `v0.2.0` release gate. +A stranger clones this repository, runs the commands in README.md, and gets a findings file for `examples/neglected-repo`. + +That is the `v0.2.0` release gate. A VM receipt is not a tag, and an implementation on `main` is not a published release. + +## Current closure + +- The clean-checkout gate runs in GitHub CI on Python 3.11 and 3.12. +- `v0.2.0` remains unpublished until the repaired head is merged and the tag is explicitly created. +- Provider base-URL overrides are operator configuration: they may come from the process environment, never from a repository `.env` file. + +## Non-goals for this handoff + +- Do not port the full skill-lib catalog. +- Do not add `--fix-one` to v0.2; remote execution and repair require later versioned work. +- Do not host a SaaS. +- Do not rewrite msdmd. +- Do not add Way / UCNS / energy text to README. + +## Prerequisite in skill-lib + +Only if `WRITE_CANON=1`: + +1. Add `status` (`runnable` | `contract` | `org-only`) and optional `runner` to `skills.json`. +2. Mark only skills that execute in-repo as `runnable`. +3. Confirm `python -m unittest discover -s tests` passes on a clean skill-lib clone. + +VM default: skip this block. Implement the inspect CLI here. Follow repo-audit-repair classes. Do not invent a sixth class. + +Allowed classes: `defect`, `environment`, `external`, `policy`, `hmmm`. + +## Target tree + +``` +pubskill-lib/ + README.md + AGENTS.md + SOURCE.md + HANDOFF.md + HANDOFF.vm.md + LICENSE + pyproject.toml + src/pubskill_lib/ + __init__.py + audit.py + schema.py + tests/ + test_audit_fixture.py + test_schema.py + examples/neglected-repo/ + README.md + pyproject.toml + .github/workflows/ci.yml + tests/test_dummy.py + expected-findings.json + .agents/skills/README.md + .agents/skills// + .github/workflows/ci.yml +``` + +## Step A — package skeleton + +Create `pyproject.toml` so `pip install -e .` works on Python 3.11+ with no extra native deps. + +Package name: `pubskill-lib` +Import name: `pubskill_lib` +Console optional: `pubskill-audit` + +Acceptance: + +```bash +python -m venv .venv && source .venv/bin/activate +python -m pip install -e . +python -c "import pubskill_lib" +``` + +## Step B — findings schema + +`findings.json` shape: + +```json +{ + "schema_version": 1, + "tool": "pubskill-lib", + "source_pin": "", + "target": { + "path": "", + "commit": "hmmm", + "remote": "hmmm", + "dirty": false + }, + "surfaces": [], + "findings": [ + { + "id": "F001", + "surface": "docs|ci|deps|identity|tests", + "claim": "", + "evidence": "", + "class": "defect", + "owner": "repository|environment|external|policy|hmmm", + "verified": false + } + ], + "hmmm": [] +} +``` -## v0.2 public boundary +`verified` defaults false. Absence of a finding is not health. -The inspect CLI accepts one local repository path: +## Step C — inspect CLI (no execute by default) ```bash python -m pubskill_lib.audit PATH --out findings.json ``` -It may inspect README files, `pyproject.toml`, `package.json`, and GitHub workflow text; record local git identity; flag missing local README targets; flag obvious test-workflow no-ops; and flag declared Python or direct local package-script entrypoints that do not exist. +v0.2 inspect only: -It does not clone remote URLs, select remote commits, install target dependencies, run target tests, repair target repositories, or expose `--fix-one`. Those capabilities require separate versioned work rather than implied v0.2 behavior. +- read README, `pyproject.toml`, `package.json`, and `.github/workflows/*` +- record identity if `.git` exists, else `hmmm` +- flag README links to missing local files or paths that escape the repository +- flag workflows that claim tests but only `echo`/no-op +- flag Python console scripts whose modules are missing +- flag direct local `package.json` script targets invoked by node/python/bash/sh when the referenced file is missing or escapes the repository +- do not install target deps +- do not run target tests -## Source provenance +Exit 0 if the tool ran. Do not exit nonzero just because the target repo is sick. Exit nonzero for tool/schema failures. -Vendored public skills must identify the same exact skill-lib SHA in both `SOURCE.md` and `.agents/skills/README.md`. Do not vendor from unpinned `main`. +## Step D — fixture -## Credential boundary +`examples/neglected-repo` must contain at least three evidenced defects the CLI will see without execution: -The examiner may read API keys and model names from `.env`, but provider base URL overrides are operator configuration and therefore come only from the process environment. Never allow an inspected repository's `.env` to redirect an ambient operator credential. +1. README references a file that does not exist +2. CI workflow named like tests that does not invoke a test runner +3. a declared script or extra path that is missing -## Release gate +Write `expected-findings.json` from the CLI output, then lock it. + +Acceptance: + +```bash +python -m pubskill_lib.audit examples/neglected-repo --out /tmp/out.json +``` + +Compare on `id`, `class`, and `surface`. + +## Step E — vendor the slice + +Optional if time or disk is scarce. Prefer A–D first. + +From skill-lib at the SOURCE.md SHA, copy only `msdmd` and `repo-audit-repair` into `.agents/skills/` and write `.agents/skills/README.md` with the same SHA. Do not copy the rest of skill-lib. + +## Step F — tests in this repo ```bash -python -m venv .venv && source .venv/bin/activate python -m pip install -e . python -m unittest discover -s tests python -m pubskill_lib.audit examples/neglected-repo --out /tmp/out.json ``` -GitHub CI executes this gate on Python 3.11 and 3.12. The first release tag is exactly `v0.2.0`. README may say the implementation is ready on `main` before that tag exists; it must not claim the release exists before the tag is published. +CI runs this gate on Python 3.11 and 3.12. The VM does not push unless `PUSH=1`. + +## Step G — close the public door + +Not a VM default. Requires `PUSH=1`. + +1. Merge only a head that passes the full gate and required review. +2. Create tag `v0.2.0` explicitly from the accepted release commit. +3. Claim the release as shipped only after the tag exists. +4. Add this repo to skill-lib consumer list only with `WRITE_CANON=1`. + +## Done / not done -## Generated metadata +Done at a verified release head: clean clone → install → unittest → audit fixture → findings.json. -`*.egg-info/` is generated build state. It is ignored and must not be tracked. Package/release verification regenerates metadata from `pyproject.toml`. +Not done: stars, SaaS, full catalog, remote execution/repair, architectural-drift theater, selling VERIFIED on the zip. ## hmmm -- Remote inspection, execution, and repair semantics are deliberately outside v0.2; version and specify them before implementation. +- Remote URL inspection, remote commit selection, target execution, and repair semantics remain outside v0.2 until separately specified. +- The fixture is continuously checked against live CLI output on `id`, `class`, and `surface`; provenance of its original byte-for-byte generation is not retained. diff --git a/README.md b/README.md index a1eccb7..050c044 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ Clone this repo when you want a command that inspects a local repository and wri ## Status -The inspect CLI works on `main`; the `v0.2.0` release tag is not published yet. +The inspect CLI implementation passes the repository gate; the `v0.2.0` release tag is not published yet. | Claim | State | |---|---| | Canon | `The-Interdependency/skill-lib` | | This repo | distribution + public CLI + fixtures | -| Clone / run / findings | **implemented on main** — release pending | +| Clone / run / findings | **implementation ready** — release pending | | VM populate | `HANDOFF.vm.md` | | Source pin | `SOURCE.md` | @@ -42,9 +42,10 @@ python -m pubskill_lib.audit PATH --out findings.json It writes: - identity when the target contains `.git` (remote, commit, dirty state) -- README links to missing local files +- README links to missing local files or paths that escape the repository - obvious test-workflow no-ops - Python `pyproject.toml` console scripts whose modules are missing +- direct local `package.json` script targets whose referenced files are missing or escape the repository - findings classified as `defect`, `environment`, `external`, `policy`, or `hmmm` The inspector does **not** yet clone URLs, select remote commits, execute target tests, or repair the target. Those are later capabilities and must not be inferred from the schema. From 78e397e1e0598b5331944f42d2cb0a1fd9445c14 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Thu, 10 Sep 2026 17:51:37 -0700 Subject: [PATCH 05/39] repair: make examiner metadata idempotent --- src/pubskill_lib/evidence.py | 47 ++++++++++++++++++++++++--- src/pubskill_lib/examine.py | 25 +++++++++------ tests/test_idempotence.py | 61 ++++++++++++++++++++++++++++++++++++ tests/test_provenance.py | 22 ++++++++++--- 4 files changed, 137 insertions(+), 18 deletions(-) create mode 100644 tests/test_idempotence.py diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index 2031a10..4f601bc 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -1,7 +1,10 @@ """Evidence engine: inventory actual code before describing it. Language comment markers are loaded from the vendored canonical msdmd parser; -this module does not maintain a second registry. +this module does not maintain a second registry. ``sha256`` is the stable source +evidence hash: generated examiner NARRATIVE blocks and RATIOS seals are excluded +so the examiner cannot make its own evidence stale. ``raw_sha256`` retains the +literal file-content hash. """ from __future__ import annotations @@ -33,6 +36,39 @@ def _comment_markers() -> dict[str, str]: return dict(markers) if isinstance(markers, dict) else {} +def source_text(text: str, marker: str | None) -> str: + """Return source text with complete generated NARRATIVE/RATIOS metadata removed. + + Trailing blank lines are normalized because RATIOS placement already removes + them. Incomplete NARRATIVE fences are preserved rather than guessed away. + """ + if marker is None: + return text + + start = f"{marker} === NARRATIVE ===" + end = f"{marker} === END NARRATIVE ===" + lines = text.splitlines() + kept: list[str] = [] + index = 0 + + while index < len(lines): + raw = lines[index] + if raw.rstrip() == start: + close = index + 1 + while close < len(lines) and lines[close].rstrip() != end: + close += 1 + if close < len(lines): + index = close + 1 + continue + if not _RATIOS_LINE_RE.match(raw.rstrip()): + kept.append(raw) + index += 1 + + while kept and not kept[-1].strip(): + kept.pop() + return "\n".join(kept) + ("\n" if kept else "") + + @dataclass class FileEvidence: path: str @@ -43,6 +79,7 @@ class FileEvidence: msdmd_blocks: dict[str, list[dict]] = field(default_factory=dict) narrative_entries: list[dict] = field(default_factory=list) sha256: str = "" + raw_sha256: str = "" size: int = 0 executable: bool = False hmmm: list[str] = field(default_factory=list) @@ -92,9 +129,11 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: item.hmmm.append("unreadable file") return item - encoded = text.encode("utf-8", errors="replace") - item.sha256 = hashlib.sha256(encoded).hexdigest() - item.size = len(encoded) + raw_encoded = text.encode("utf-8", errors="replace") + stable_encoded = source_text(text, marker).encode("utf-8", errors="replace") + item.raw_sha256 = hashlib.sha256(raw_encoded).hexdigest() + item.sha256 = hashlib.sha256(stable_encoded).hexdigest() + item.size = len(raw_encoded) try: item.executable = bool(path.stat().st_mode & 0o111) except OSError: diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index 82c8d4c..a2e88ab 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -58,19 +58,18 @@ def _apply( for ev in evidence_list: path = boundary.assert_inside(root, root / ev.path) - text = _read_text(path) + original_text = _read_text(path) if ev.marker is None: continue - new_text = text - engine = ratios.RatiosEngine() - values = engine.compute(path, text) - new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) - if ratio_changed: - changed.append(f"{ev.path}:ratios") - + new_text = original_text if narrate: - result = narrative.narrate_file(ev, text, provider_list, now) + result = narrative.narrate_file( + ev, + evidence.source_text(original_text, ev.marker), + provider_list, + now, + ) narratives[ev.path] = result.entry if result.hmmm: result.entry["summary"] = result.entry["summary"] or "hmmm" @@ -87,7 +86,13 @@ def _apply( if block_changed: changed.append(f"{ev.path}:narrative") - if new_text != text: + engine = ratios.RatiosEngine() + values = engine.compute(path, new_text) + new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) + if ratio_changed: + changed.append(f"{ev.path}:ratios") + + if new_text != original_text: msdmd_writer.write_text_safely(path, new_text) return narratives, {"changed": changed, "narrated": len(narratives)} diff --git a/tests/test_idempotence.py b/tests/test_idempotence.py new file mode 100644 index 0000000..170a1f8 --- /dev/null +++ b/tests/test_idempotence.py @@ -0,0 +1,61 @@ +import tempfile +import unittest +from pathlib import Path + +from pubskill_lib import evidence, examine, narrative + + +class ExaminerIdempotenceTests(unittest.TestCase): + def test_generated_metadata_does_not_stale_its_own_narrative(self): + class FakeProvider: + name = "fake" + model = "model-1" + + def chat(self, system, user): + return "Prints a greeting." + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "tool.py" + source.write_text( + "#!/usr/bin/env python3\n" + "# -*- coding: utf-8 -*-\n" + "print('hi')\n", + encoding="utf-8", + ) + + first_evidence = evidence.inventory(root) + _, first_report = examine._apply(root, first_evidence, [FakeProvider()], True) + first_output = source.read_text(encoding="utf-8") + self.assertTrue(first_report["changed"]) + + second_evidence = evidence.inventory(root) + self.assertEqual(1, len(second_evidence)) + entry = second_evidence[0].narrative_entries[0] + self.assertFalse(narrative.is_stale(entry, second_evidence[0].sha256)) + + _, second_report = examine._apply(root, second_evidence, [], False) + second_output = source.read_text(encoding="utf-8") + self.assertEqual(first_output, second_output) + self.assertEqual([], second_report["changed"]) + + def test_source_hash_excludes_generated_narrative_and_ratios(self): + plain = "print('hi')\n" + decorated = ( + "# ratios: loc_comments=1:0 imports_exports=0:0 calls_definitions=1:0\n" + "# === NARRATIVE ===\n" + "# id: examiner_x\n" + "# summary: Prints hi.\n" + "# evidence_sha256: x\n" + "# === END NARRATIVE ===\n" + "print('hi')\n" + "# ratios: loc_comments=1:0 imports_exports=0:0 calls_definitions=1:0\n" + ) + self.assertEqual( + evidence.source_text(plain, "#"), + evidence.source_text(decorated, "#"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_provenance.py b/tests/test_provenance.py index c941f15..f1aebac 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -1,3 +1,4 @@ +import json import re import unittest from pathlib import Path @@ -7,15 +8,28 @@ PIN_RE = re.compile(r"`([0-9a-f]{40})`") +def _source_pin() -> str: + source = (REPO / "SOURCE.md").read_text(encoding="utf-8") + match = PIN_RE.search(source) + if match is None: + raise AssertionError("SOURCE.md has no pinned SHA") + return match.group(1) + + class PublicationProvenanceTests(unittest.TestCase): def test_source_pin_matches_vendored_skill_manifest(self): - source = (REPO / "SOURCE.md").read_text(encoding="utf-8") vendored = (REPO / ".agents" / "skills" / "README.md").read_text(encoding="utf-8") - source_pin = PIN_RE.search(source) vendored_pin = PIN_RE.search(vendored) - self.assertIsNotNone(source_pin) self.assertIsNotNone(vendored_pin) - self.assertEqual(source_pin.group(1), vendored_pin.group(1)) + self.assertEqual(_source_pin(), vendored_pin.group(1)) + + def test_fixture_source_pin_matches_publication_pin(self): + expected = json.loads( + (REPO / "examples" / "neglected-repo" / "expected-findings.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(_source_pin(), expected["source_pin"]) def test_local_secret_files_are_ignored(self): ignore = (REPO / ".gitignore").read_text(encoding="utf-8").splitlines() From c7b2f05c77562d46430ceb1b72316a88477ef9d8 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Thu, 10 Sep 2026 17:54:02 -0700 Subject: [PATCH 06/39] repair: retain numeric msdmd narrative fields --- src/pubskill_lib/evidence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index 4f601bc..0e3d7a4 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -96,7 +96,7 @@ def _block_name_re(marker: str) -> re.Pattern[str]: def _parse_block_entries(marker: str, body: str) -> list[dict]: m = re.escape(marker) id_re = re.compile(rf"^\s*{m}\s*id:\s*(?P\S+)\s*$") - field_re = re.compile(rf"^\s*{m}\s+(?P[a-z_]+):\s*(?P.+?)\s*$") + field_re = re.compile(rf"^\s*{m}\s+(?P[a-z][a-z0-9_]*):\s*(?P.+?)\s*$") entries: list[dict[str, str]] = [] current: dict[str, str] | None = None for line in body.splitlines(): From a1681cff1014aadc1ff84e81538850ca08a4b318 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Thu, 10 Sep 2026 17:56:10 -0700 Subject: [PATCH 07/39] repair: make narrative upsert idempotent and harden CI --- .github/workflows/ci.yml | 8 +++++-- src/pubskill_lib/msdmd_writer.py | 38 ++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 293b507..b24b587 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,7 @@ name: ci on: [push, pull_request] +permissions: + contents: read jobs: test: runs-on: ubuntu-latest @@ -8,8 +10,10 @@ jobs: matrix: python-version: ['3.11', '3.12'] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - run: | diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 1c76ea4..491b240 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -7,7 +7,6 @@ from __future__ import annotations -import re from pathlib import Path from . import ratios @@ -15,14 +14,6 @@ NARRATIVE_BLOCK = "NARRATIVE" -def _fence_re(marker: str) -> re.Pattern[str]: - m = re.escape(marker) - return re.compile( - rf"^{m} === {NARRATIVE_BLOCK} ===\s*$.*?^{m} === END {NARRATIVE_BLOCK} ===\s*$", - re.MULTILINE | re.DOTALL, - ) - - def _block_lines(marker: str, entry: dict[str, str]) -> list[str]: lines = [f"{marker} === {NARRATIVE_BLOCK} ==="] for key, value in entry.items(): @@ -31,6 +22,30 @@ def _block_lines(marker: str, entry: dict[str, str]) -> list[str]: return lines +def _without_narrative_lines(text: str, marker: str) -> list[str]: + """Remove complete NARRATIVE blocks without creating phantom blank lines.""" + start = f"{marker} === {NARRATIVE_BLOCK} ===" + end = f"{marker} === END {NARRATIVE_BLOCK} ===" + lines = text.splitlines() + kept: list[str] = [] + index = 0 + + while index < len(lines): + if lines[index].rstrip() == start: + close = index + 1 + while close < len(lines) and lines[close].rstrip() != end: + close += 1 + if close < len(lines): + index = close + 1 + continue + kept.append(lines[index]) + index += 1 + + while kept and not kept[-1].strip(): + kept.pop() + return kept + + def narrative_id(sha256: str) -> str: return f"examiner_{sha256[:16]}" @@ -42,9 +57,8 @@ def upsert_narrative( path: Path | None = None, ) -> tuple[str, bool]: """Replace NARRATIVE blocks without crossing the protected opening boundary.""" - body = _fence_re(marker).sub("", text).rstrip("\n") + lines = _without_narrative_lines(text, marker) block = "\n".join(_block_lines(marker, entry)) - lines = body.splitlines() adapter = ratios.RatiosEngine().adapter_for(path) if path is not None else None insert_at = ratios.opening_index(lines, adapter) @@ -53,7 +67,7 @@ def upsert_narrative( insert_at += 1 lines.insert(insert_at, block) - new_text = "\n".join(lines) + "\n" + new_text = "\n".join(lines) + ("\n" if lines else "") return new_text, new_text != text From 1373ea47784d473cd7fcebe18758295af2a9ad8b Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:34:41 -0700 Subject: [PATCH 08/39] fix: treat blank model overrides as unset --- src/pubskill_lib/providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pubskill_lib/providers.py b/src/pubskill_lib/providers.py index ca021e2..59ae102 100644 --- a/src/pubskill_lib/providers.py +++ b/src/pubskill_lib/providers.py @@ -61,7 +61,7 @@ def __init__(self, name: str, env: dict[str, str]): self.name = name self.env = env self.key = env.get(self.key_env(), "") - self.model = env.get(self.model_env(), self.default_model()) + self.model = env.get(self.model_env()) or self.default_model() def key_env(self) -> str: raise NotImplementedError From e528267780c3c2ca47fbf440ebfdb544493152c6 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:35:05 -0700 Subject: [PATCH 09/39] fix: mutate only safe ratio languages --- src/pubskill_lib/examine.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index a2e88ab..57032ef 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -55,11 +55,13 @@ def _apply( narratives: dict[str, dict[str, str]] = {} changed: list[str] = [] now = datetime.now(timezone.utc).isoformat() + engine = ratios.RatiosEngine() for ev in evidence_list: path = boundary.assert_inside(root, root / ev.path) original_text = _read_text(path) - if ev.marker is None: + adapter = engine.adapter_for(path) + if ev.marker is None or adapter is None: continue new_text = original_text @@ -86,8 +88,7 @@ def _apply( if block_changed: changed.append(f"{ev.path}:narrative") - engine = ratios.RatiosEngine() - values = engine.compute(path, new_text) + values = engine.compute(path, evidence.source_text(new_text, ev.marker)) new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) if ratio_changed: changed.append(f"{ev.path}:ratios") From 5085f77947bfc6e473ea427d19a6134258eeca84 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:35:48 -0700 Subject: [PATCH 10/39] fix: close package-script audit gaps --- src/pubskill_lib/audit.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index a8eba6f..7c33ecd 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -10,6 +10,7 @@ import argparse import json import re +import shlex import subprocess import sys from pathlib import Path @@ -26,9 +27,8 @@ ECHO_OR_NOOP_PATTERN = re.compile(r"\b(echo|true|exit\s+0|printf)\b", re.IGNORECASE) MARKDOWN_LINK_PATTERN = re.compile(r"\[[^\]]*\]\(([^)]+)\)") PIN_PATTERN = re.compile(r"`([0-9a-f]{40})`") -LOCAL_SCRIPT_PATTERN = re.compile( - r"(?:^|(?:&&|;|\|)\s*)(?:node|python(?:3)?|bash|sh)\s+([^\s;&|]+)" -) +LOCAL_SCRIPT_INTERPRETERS = {"node", "python", "python3", "bash", "sh"} +NON_FILE_MODES = {"-c", "-m", "-e", "--eval", "--print", "-p"} class _Sink: @@ -161,6 +161,29 @@ def _check_pyproject_scripts(target, sink): ) +def _local_script_targets(command): + """Yield direct local script operands without mistaking interpreter flags for paths.""" + for segment in re.split(r"\s*(?:&&|;|\|)\s*", command): + if not segment.strip(): + continue + try: + tokens = shlex.split(segment) + except ValueError: + continue + if not tokens or tokens[0] not in LOCAL_SCRIPT_INTERPRETERS: + continue + index = 1 + while index < len(tokens): + token = tokens[index] + if token in NON_FILE_MODES: + break + if token.startswith("-"): + index += 1 + continue + yield token + break + + def _check_package_scripts(target, sink): package = target / "package.json" text = _read_text(package) @@ -171,15 +194,18 @@ def _check_package_scripts(target, sink): except json.JSONDecodeError: sink.add("deps", "package.json is not valid JSON", "package.json") return + if not isinstance(data, dict): + sink.add("deps", "package.json top level is not an object", "package.json") + return scripts = data.get("scripts") or {} if not isinstance(scripts, dict): return for name, command in sorted(scripts.items()): if not isinstance(command, str): continue - for match in LOCAL_SCRIPT_PATTERN.finditer(command): - raw_path = match.group(1).strip('"\'') - if raw_path.startswith(("-", "/")) or "://" in raw_path: + for raw_path in _local_script_targets(command): + raw_path = raw_path.strip('"\'') + if raw_path.startswith("/") or "://" in raw_path: continue local = (target / raw_path).resolve() try: From b0b6afa90a8bdbd3c21db1e690408d142f39a7e4 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:36:41 -0700 Subject: [PATCH 11/39] test: cover repaired audit and examiner boundaries --- tests/test_repairs.py | 66 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/tests/test_repairs.py b/tests/test_repairs.py index d26a46b..e5027b6 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -7,7 +7,7 @@ from pathlib import Path from unittest.mock import patch -from pubskill_lib import audit, evidence, msdmd_writer, narrative, providers +from pubskill_lib import audit, evidence, examine, msdmd_writer, narrative, providers, ratios class CredentialBoundaryTests(unittest.TestCase): @@ -38,6 +38,12 @@ def test_process_environment_may_set_base_url(self): merged = providers.env_with_dotenv(env_file) self.assertEqual("https://operator.example/v1", merged["OPENAI_BASE_URL"]) + def test_blank_model_override_uses_provider_default(self): + openai = providers.OpenAIProvider({"OPENAI_API_KEY": "key", "OPENAI_MODEL": ""}) + anthropic = providers.AnthropicProvider({"ANTHROPIC_API_KEY": "key", "ANTHROPIC_MODEL": ""}) + self.assertEqual(openai.default_model(), openai.model) + self.assertEqual(anthropic.default_model(), anthropic.model) + class NarrativeBoundaryTests(unittest.TestCase): def test_narrative_preserves_python_shebang_and_encoding_header(self): @@ -78,6 +84,39 @@ def chat(self, system, user): self.assertEqual("fake", result.entry["provider"]) self.assertEqual("model-1", result.entry["model"]) + def test_generated_narrative_does_not_change_ratios(self): + class FakeProvider: + name = "fake" + model = "model-1" + + def chat(self, system, user): + return "Generated prose mentions fake_call() and comments." + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "x.py" + original = "print('x')\n" + path.write_text(original, encoding="utf-8") + ev = evidence.read_evidence(root, path) + expected = ratios.RatiosEngine().compute(path, original) + examine._apply(root, [ev], [FakeProvider()], True) + written = evidence.read_evidence(root, path) + self.assertTrue(written.ratios_lines) + for key, value in expected.items(): + self.assertIn(f"{key}={value}", written.ratios_lines[0]) + + def test_apply_skips_parser_supported_language_without_safe_ratio_adapter(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "index.php" + original = " Date: Fri, 11 Sep 2026 05:43:05 -0700 Subject: [PATCH 12/39] chore: repin canonical msdmd parser --- .agents/skills/msdmd/parsers/universal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.agents/skills/msdmd/parsers/universal.py b/.agents/skills/msdmd/parsers/universal.py index 204f533..e6cb88a 100644 --- a/.agents/skills/msdmd/parsers/universal.py +++ b/.agents/skills/msdmd/parsers/universal.py @@ -100,7 +100,7 @@ def parse_text(text: str, block_name: str, marker: str = "#") -> list[dict]: block_re = _block_regex(block_name, marker) m = re.escape(marker) id_re = re.compile(rf"^\s*{m}\s*id:\s*(?P\S+)\s*$") - field_re = re.compile(rf"^\s*{m}\s+(?P[a-z_]+):\s*(?P.+?)\s*$") + field_re = re.compile(rf"^\s*{m}\s+(?P[a-z_][a-z0-9_]*):\s*(?P.+?)\s*$") entries: list[dict] = [] for block in block_re.finditer(text): @@ -189,7 +189,7 @@ def iter_source_files(path: Path) -> Iterable[Path]: # boundary to literal line 2: # ratios: loc_comments=N:M imports_exports=N:M calls_definitions=N:M RATIO_IDS = ("loc_comments", "imports_exports", "calls_definitions") -_RATIOS_TOKEN_RE = re.compile(r"(?P[a-z_]+)=(?P\S+)") +_RATIOS_TOKEN_RE = re.compile(r"(?P[a-z_][a-z0-9_]*)=(?P\S+)") def _ratios_line_re(marker: str) -> re.Pattern[str]: @@ -253,4 +253,4 @@ def ratios_placement(text: str, marker: str = "#") -> tuple[bool, bool]: last_ok = bool(line_re.match(raw.rstrip())) break return (opening_ok, last_ok) -# ratios: loc_comments=161:57 imports_exports=4:7 calls_definitions=55:10 +# ratios: loc_comments=161:57 imports_exports=4:7 calls_definitions=55:10 \ No newline at end of file From 953765e47c115f7219968f9cd148354aee1151d5 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:43:47 -0700 Subject: [PATCH 13/39] chore: align TypeScript parser with canonical msdmd --- .agents/skills/msdmd/parsers/universal.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.agents/skills/msdmd/parsers/universal.ts b/.agents/skills/msdmd/parsers/universal.ts index 141f46c..2d3f3c2 100644 --- a/.agents/skills/msdmd/parsers/universal.ts +++ b/.agents/skills/msdmd/parsers/universal.ts @@ -86,7 +86,7 @@ export function parseText( "gm", ); const idRe = new RegExp(`^\\s*${m}\\s*id:\\s*(\\S+)\\s*$`); - const fieldRe = new RegExp(`^\\s*${m}\\s+([a-z_]+):\\s*(.+?)\\s*$`); + const fieldRe = new RegExp(`^\\s*${m}\\s+([a-z_][a-z0-9_]*):\\s*(.+?)\\s*$`); const entries: Entry[] = []; let match: RegExpExecArray | null; @@ -185,7 +185,7 @@ function ratiosLineRe(marker: string): RegExp { export function parseRatios(text: string, marker: string = "#"): Entry[] { const lineRe = ratiosLineRe(marker); - const tokenRe = /([a-z_]+)=(\S+)/g; + const tokenRe = /([a-z_][a-z0-9_]*)=(\S+)/g; const out: Entry[] = []; for (const raw of text.split("\n")) { const lm = lineRe.exec(raw.replace(/\s+$/, "")); @@ -231,4 +231,4 @@ export function ratiosPlacement(text: string, marker: string = "#"): [boolean, b } return [openingOk && !displacedShebang, lastOk]; } -// ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm +// ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm \ No newline at end of file From e82f92538fa8d92115382f5516fd7e7e23c62642 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:44:36 -0700 Subject: [PATCH 14/39] chore: preserve canonical parser bytes --- .agents/skills/msdmd/parsers/universal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/skills/msdmd/parsers/universal.ts b/.agents/skills/msdmd/parsers/universal.ts index 2d3f3c2..67b7b21 100644 --- a/.agents/skills/msdmd/parsers/universal.ts +++ b/.agents/skills/msdmd/parsers/universal.ts @@ -231,4 +231,4 @@ export function ratiosPlacement(text: string, marker: string = "#"): [boolean, b } return [openingOk && !displacedShebang, lastOk]; } -// ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm \ No newline at end of file +// ratios: loc_comments=hmmm imports_exports=hmmm calls_definitions=hmmm From edc29c050fb50edd0585ccf29d038356b1a291ab Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:45:31 -0700 Subject: [PATCH 15/39] chore: refresh canonical msdmd skill --- .agents/skills/msdmd/SKILL.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.agents/skills/msdmd/SKILL.md b/.agents/skills/msdmd/SKILL.md index ffa10c4..1a6f0be 100644 --- a/.agents/skills/msdmd/SKILL.md +++ b/.agents/skills/msdmd/SKILL.md @@ -63,7 +63,9 @@ claims to prove those obligations. See referenced from external tooling). - **Field lines**: indented one level beneath the id (two spaces of visible indent inside the comment). Field names are lowercase - snake_case followed by `:` and a value. + snake_case followed by `:` and a value. Digits are allowed after the first + character, so `evidence_sha256` is valid; the first character must be a + lowercase letter or underscore. - **Multiple blocks per file**: a module may declare more than one block, of the same or different types. The parser concatenates entries. From 76fa0807c4392320569eae63f5fb2d2f17da5ac0 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:45:42 -0700 Subject: [PATCH 16/39] docs: pin merged skill-lib authority --- SOURCE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SOURCE.md b/SOURCE.md index 994533e..ed6f935 100644 --- a/SOURCE.md +++ b/SOURCE.md @@ -4,8 +4,8 @@ Canon: https://github.com/The-Interdependency/skill-lib | Field | Value | |---|---| -| Pinned SHA | `c14ee9d500579a4b5d6821f62c9d82ca96e73608` | -| Pinned date | 2026-09-06 | +| Pinned SHA | `8de4f12d0f31ff94f41e4a0196c447c0cbe20faf` | +| Pinned date | 2026-09-11 | | Pin meaning | exact canonical source used for the vendored public skill slice | | Runnable subset | `msdmd` (runnable), `repo-audit-repair` (contract) | From cc2e8ca40bff2aaa80c8b282b5b3101d11688f1f Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:45:55 -0700 Subject: [PATCH 17/39] docs: record refreshed canonical skill source --- .agents/skills/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/skills/README.md b/.agents/skills/README.md index a009e69..387f291 100644 --- a/.agents/skills/README.md +++ b/.agents/skills/README.md @@ -3,7 +3,7 @@ This directory contains repo-local copies of canonical skills from `The-Interdependency/skill-lib`. -Source commit: `c14ee9d500579a4b5d6821f62c9d82ca96e73608` +Source commit: `8de4f12d0f31ff94f41e4a0196c447c0cbe20faf` Repo-local copies are not the source of truth. Edit `skill-lib` first, then propagate from the canonical source. From 4e7c7010d1ccfa860c0a375e0809c6bf6e131636 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:46:33 -0700 Subject: [PATCH 18/39] fix: consume canonical msdmd entry parser --- src/pubskill_lib/evidence.py | 65 +++++++++++++----------------------- 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index 0e3d7a4..628ad55 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -1,10 +1,10 @@ """Evidence engine: inventory actual code before describing it. -Language comment markers are loaded from the vendored canonical msdmd parser; -this module does not maintain a second registry. ``sha256`` is the stable source -evidence hash: generated examiner NARRATIVE blocks and RATIOS seals are excluded -so the examiner cannot make its own evidence stale. ``raw_sha256`` retains the -literal file-content hash. +Language comment markers and entry grammar are loaded from the vendored +canonical msdmd parser; this module does not maintain a second dialect. +``sha256`` is the stable source evidence hash: generated examiner NARRATIVE +blocks and RATIOS seals are excluded so the examiner cannot make its own +evidence stale. ``raw_sha256`` retains the literal file-content hash. """ from __future__ import annotations @@ -23,16 +23,22 @@ @lru_cache(maxsize=1) -def _comment_markers() -> dict[str, str]: - """Load COMMENT_MARKERS from the pinned repo-local msdmd parser.""" +def _msdmd_parser(): + """Load the pinned repo-local canonical msdmd parser module.""" repo = Path(__file__).resolve().parents[2] parser_path = repo / ".agents" / "skills" / "msdmd" / "parsers" / "universal.py" spec = importlib.util.spec_from_file_location("pubskill_lib._vendored_msdmd", parser_path) if spec is None or spec.loader is None: - return {} + raise RuntimeError(f"cannot load vendored msdmd parser: {parser_path}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - markers = getattr(module, "COMMENT_MARKERS", {}) + return module + + +@lru_cache(maxsize=1) +def _comment_markers() -> dict[str, str]: + """Load COMMENT_MARKERS from the pinned repo-local msdmd parser.""" + markers = getattr(_msdmd_parser(), "COMMENT_MARKERS", {}) return dict(markers) if isinstance(markers, dict) else {} @@ -85,36 +91,11 @@ class FileEvidence: hmmm: list[str] = field(default_factory=list) -def _block_name_re(marker: str) -> re.Pattern[str]: - m = re.escape(marker) - return re.compile( - rf"^{m} === ([A-Z_]+) ===\s*$(?P.*?)^{m} === END \1 ===\s*$", - re.MULTILINE | re.DOTALL, - ) - - -def _parse_block_entries(marker: str, body: str) -> list[dict]: +def _block_names(text: str, marker: str) -> list[str]: + """Return distinct declared block names; canonical parser owns entry grammar.""" m = re.escape(marker) - id_re = re.compile(rf"^\s*{m}\s*id:\s*(?P\S+)\s*$") - field_re = re.compile(rf"^\s*{m}\s+(?P[a-z][a-z0-9_]*):\s*(?P.+?)\s*$") - entries: list[dict[str, str]] = [] - current: dict[str, str] | None = None - for line in body.splitlines(): - line = line.rstrip() - match_id = id_re.match(line) - if match_id: - if current is not None: - entries.append(current) - current = {"id": match_id.group("id")} - continue - if current is None: - continue - match_field = field_re.match(line) - if match_field: - current[match_field.group("key")] = match_field.group("val") - if current is not None: - entries.append(current) - return entries + start_re = re.compile(rf"^{m} === (?P[A-Z_]+) ===\s*$", re.MULTILINE) + return list(dict.fromkeys(match.group("name") for match in start_re.finditer(text))) def read_evidence(root: Path, path: Path) -> FileEvidence: @@ -147,10 +128,10 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: for raw in text.splitlines(): if _RATIOS_LINE_RE.match(raw.rstrip()): item.ratios_lines.append(raw.rstrip()) - for match in _block_name_re(marker).finditer(text): - name = match.group(1) - entries = _parse_block_entries(marker, match.group("body")) - item.msdmd_blocks.setdefault(name, []).extend(entries) + parser = _msdmd_parser() + for name in _block_names(text, marker): + entries = parser.parse_text(text, name, marker) + item.msdmd_blocks[name] = entries item.narrative_entries = item.msdmd_blocks.get("NARRATIVE", []) else: item.hmmm.append(f"unsupported language for msdmd: .{language}") From 3d52e9f6e43ea80623a716bd2adeb6e82e423191 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 05:59:36 -0700 Subject: [PATCH 19/39] fix audit interpreter and path boundaries --- src/pubskill_lib/audit.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 7c33ecd..7978965 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -28,7 +28,13 @@ MARKDOWN_LINK_PATTERN = re.compile(r"\[[^\]]*\]\(([^)]+)\)") PIN_PATTERN = re.compile(r"`([0-9a-f]{40})`") LOCAL_SCRIPT_INTERPRETERS = {"node", "python", "python3", "bash", "sh"} -NON_FILE_MODES = {"-c", "-m", "-e", "--eval", "--print", "-p"} +NON_FILE_MODES = { + "node": {"-e", "--eval", "-p", "--print"}, + "python": {"-c", "-m"}, + "python3": {"-c", "-m"}, + "bash": {"-c"}, + "sh": {"-c"}, +} class _Sink: @@ -162,7 +168,7 @@ def _check_pyproject_scripts(target, sink): def _local_script_targets(command): - """Yield direct local script operands without mistaking interpreter flags for paths.""" + """Yield direct local script operands without confusing interpreter modes with flags.""" for segment in re.split(r"\s*(?:&&|;|\|)\s*", command): if not segment.strip(): continue @@ -172,10 +178,12 @@ def _local_script_targets(command): continue if not tokens or tokens[0] not in LOCAL_SCRIPT_INTERPRETERS: continue + interpreter = tokens[0] + non_file_modes = NON_FILE_MODES[interpreter] index = 1 while index < len(tokens): token = tokens[index] - if token in NON_FILE_MODES: + if token in non_file_modes: break if token.startswith("-"): index += 1 @@ -205,9 +213,10 @@ def _check_package_scripts(target, sink): continue for raw_path in _local_script_targets(command): raw_path = raw_path.strip('"\'') - if raw_path.startswith("/") or "://" in raw_path: + if "://" in raw_path: continue - local = (target / raw_path).resolve() + candidate = Path(raw_path) + local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() try: local.relative_to(target.resolve()) except ValueError: From 3b9162d56a7c1b5b5481e8fbf2a90c989f83cbee Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 06:00:32 -0700 Subject: [PATCH 20/39] package canonical msdmd parser --- src/pubskill_lib/_msdmd_universal.py | 256 +++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 src/pubskill_lib/_msdmd_universal.py diff --git a/src/pubskill_lib/_msdmd_universal.py b/src/pubskill_lib/_msdmd_universal.py new file mode 100644 index 0000000..e6cb88a --- /dev/null +++ b/src/pubskill_lib/_msdmd_universal.py @@ -0,0 +1,256 @@ +# ratios: loc_comments=161:57 imports_exports=4:7 calls_definitions=55:10 +"""Universal msdmd parser — pure stdlib. + +Implements the parser contract from ``msdmd/SKILL.md``: extracts every +``# === ===`` … ``# === END ===`` block from +a source file and returns its entries as flat dicts. + +Comment marker is auto-detected by file extension. The block syntax +itself is identical across languages; only the per-line marker changes. +``COMMENT_MARKERS`` is public so runners can distinguish parser support +from their narrower language-specific execution or metric coverage. + +Public API: + + parse_text(text, block_name, marker="#") -> list[dict] + parse_file(path, block_name) -> list[dict] + walk_tree(root, block_name, *, skip=None, extensions=None) -> tuple[annotated, untested] + +RATIOS is the one msdmd declaration that is *not* a fenced block — it is a +single comment line carried on a file's opening and closing source boundaries. +A valid interpreter shebang owns literal line 1, so the opening RATIOS line is +literal line 2 in that case. The reader lives here as a sanctioned extension: + + parse_ratios(text, marker="#") -> list[dict] + parse_ratios_file(path) -> list[dict] + ratios_placement(text, marker="#") -> tuple[opening_ok, closing_ok] + +This module has zero non-stdlib dependencies and is safe to copy +verbatim into any project that wants msdmd support. +""" +from __future__ import annotations +import re +from pathlib import Path +from typing import Iterable + +# extension → line-comment marker. Keep this registry entry-for-entry equivalent +# to universal.ts; tests fail if either parser gains or loses an extension alone. +COMMENT_MARKERS: dict[str, str] = { + ".py": "#", ".pyw": "#", ".pyi": "#", + ".rb": "#", ".rake": "#", ".gemspec": "#", + ".ex": "#", ".exs": "#", + ".sh": "#", ".bash": "#", ".zsh": "#", ".fish": "#", + ".pl": "#", ".pm": "#", ".t": "#", + ".r": "#", ".jl": "#", + ".ps1": "#", ".psm1": "#", ".tcl": "#", + ".raku": "#", ".rakumod": "#", + ".ts": "//", ".tsx": "//", ".mts": "//", ".cts": "//", + ".js": "//", ".jsx": "//", ".mjs": "//", ".cjs": "//", + ".rs": "//", ".go": "//", ".java": "//", + ".c": "//", ".cc": "//", ".cp": "//", ".cpp": "//", + ".cxx": "//", ".c+": "//", ".c++": "//", + ".h": "//", ".hh": "//", ".hp": "//", ".hpp": "//", + ".hxx": "//", ".h+": "//", ".h++": "//", + ".tcc": "//", ".ipp": "//", ".inl": "//", + ".swift": "//", ".kt": "//", ".kts": "//", ".cs": "//", + ".mm": "//", ".scala": "//", ".dart": "//", ".zig": "//", + ".groovy": "//", ".gradle": "//", ".php": "//", + ".sql": "--", ".lua": "--", ".hs": "--", + ".adb": "--", ".ads": "--", ".vhd": "--", ".vhdl": "--", + ".lean": "--", + ".erl": "%", ".hrl": "%", ".prolog": "%", + ".clj": ";", ".cljs": ";", ".cljc": ";", ".bb": ";", + ".lisp": ";", ".lsp": ";", ".cl": ";", + ".scm": ";", ".ss": ";", ".rkt": ";", + ".f": "!", ".for": "!", ".f90": "!", ".f95": "!", + ".f03": "!", ".f08": "!", + ".vb": "'", ".vbs": "'", + ".cob": "*>", ".cbl": "*>", +} + +_DEFAULT_SKIP = ( + "__pycache__", "node_modules", ".git", ".venv", "venv", + "dist", "build", ".next", ".nuxt", "target", ".pytest_cache", + ".mypy_cache", ".tox", +) + + +def marker_for(path: Path) -> str | None: + """Return the comment marker for a file path, or None if unsupported.""" + return COMMENT_MARKERS.get(path.suffix.lower()) + + +def _block_regex(block_name: str, marker: str) -> re.Pattern[str]: + m = re.escape(marker) + name = re.escape(block_name) + return re.compile( + rf"^{m} === {name} ===\s*$(?P.*?)^{m} === END {name} ===\s*$", + re.MULTILINE | re.DOTALL, + ) + + +def parse_text(text: str, block_name: str, marker: str = "#") -> list[dict]: + """Extract every entry from every matching block in ``text``. + + Entries are flat ``dict[str, str]`` keyed by field name. The first + line of an entry must be ``id: ``; subsequent lines until + the next ``id:`` (or block end) carry indented ``: `` + pairs. + """ + block_re = _block_regex(block_name, marker) + m = re.escape(marker) + id_re = re.compile(rf"^\s*{m}\s*id:\s*(?P\S+)\s*$") + field_re = re.compile(rf"^\s*{m}\s+(?P[a-z_][a-z0-9_]*):\s*(?P.+?)\s*$") + + entries: list[dict] = [] + for block in block_re.finditer(text): + current: dict[str, str] | None = None + for line in block.group("body").splitlines(): + line = line.rstrip() + mid = id_re.match(line) + if mid: + if current is not None: + entries.append(current) + current = {"id": mid.group("id")} + continue + if current is None: + continue + mf = field_re.match(line) + if mf: + current[mf.group("key")] = mf.group("val") + if current is not None: + entries.append(current) + return entries + + +def parse_file(path: Path, block_name: str) -> list[dict]: + """Parse a single file. Returns [] if the file's extension has no + known comment marker or if the file can't be read.""" + marker = marker_for(path) + if marker is None: + return [] + try: + return parse_text(path.read_text(encoding="utf-8"), block_name, marker) + except (OSError, UnicodeDecodeError): + return [] + + +def walk_tree( + root: Path, + block_name: str, + *, + skip: Iterable[str] | None = None, + extensions: Iterable[str] | None = None, +) -> tuple[list[tuple[Path, list[dict]]], list[Path]]: + """Walk ``root`` and partition source files into (annotated, untested). + + ``annotated`` is a list of ``(path, entries)`` for every file that + contains at least one entry of ``block_name``. ``untested`` is every + other source file (still filtered by extension and skip-dirs) so + coverage gaps remain observable. + """ + skip_set = set(skip) if skip is not None else set(_DEFAULT_SKIP) + ext_set = ( + set(e.lower() if e.startswith(".") else "." + e.lower() for e in extensions) + if extensions is not None + else set(COMMENT_MARKERS.keys()) + ) + + def iter_source_files(path: Path) -> Iterable[Path]: + if path.name in skip_set: + return + try: + children = sorted(path.iterdir()) + except OSError: + return + for child in children: + if child.is_dir(): + if child.name in skip_set: + continue + yield from iter_source_files(child) + elif child.is_file() and child.suffix.lower() in ext_set: + yield child + + annotated: list[tuple[Path, list[dict]]] = [] + untested: list[Path] = [] + for path in iter_source_files(root): + entries = parse_file(path, block_name) + if entries: + annotated.append((path, entries)) + else: + untested.append(path) + return annotated, untested + + +# --- RATIOS single-line declaration (msdmd extension) -------------------- +# Unlike every other declaration, RATIOS is not fenced. It is a single +# comment line carrying the three canonical ratios at the opening source +# boundary and last non-blank line. A valid line-1 shebang moves the opening +# boundary to literal line 2: +# ratios: loc_comments=N:M imports_exports=N:M calls_definitions=N:M +RATIO_IDS = ("loc_comments", "imports_exports", "calls_definitions") +_RATIOS_TOKEN_RE = re.compile(r"(?P[a-z_][a-z0-9_]*)=(?P\S+)") + + +def _ratios_line_re(marker: str) -> re.Pattern[str]: + return re.compile(rf"^{re.escape(marker)}\s*ratios:\s*(?P.+?)\s*$") + + +def parse_ratios(text: str, marker: str = "#") -> list[dict]: + """Read single-line RATIOS declarations from ``text``. + + RATIOS is not a fenced block: it is one comment line of the form + `` ratios: loc_comments=N:M imports_exports=N:M calls_definitions=N:M`` + placed at the file's opening and closing source boundaries. Returns one + flat ``{"id", "value"}`` dict per (declaration line x ratio token) so a + drift gate can verify every occurrence. + """ + line_re = _ratios_line_re(marker) + out: list[dict] = [] + for raw in text.splitlines(): + lm = line_re.match(raw.rstrip()) + if not lm: + continue + for tm in _RATIOS_TOKEN_RE.finditer(lm.group("body")): + out.append({"id": tm.group("key"), "value": tm.group("val")}) + return out + + +def parse_ratios_file(path: Path) -> list[dict]: + """``parse_ratios`` for a file path (marker auto-detected); [] on error.""" + marker = marker_for(path) + if marker is None: + return [] + try: + return parse_ratios(path.read_text(encoding="utf-8"), marker) + except (OSError, UnicodeDecodeError): + return [] + + +def ratios_placement(text: str, marker: str = "#") -> tuple[bool, bool]: + """Return ``(opening_ratios_ok, closing_ratios_ok)``. + + A non-empty ``#!`` interpreter directive may occupy literal line 1. It is + the only accepted preamble and RATIOS must immediately follow it. + """ + line_re = _ratios_line_re(marker) + lines = text.splitlines() + if not lines: + return (False, False) + has_shebang = lines[0].startswith("#!") and bool(lines[0][2:].strip()) + opening_index = 1 if has_shebang else 0 + opening_ok = ( + len(lines) > opening_index + and bool(line_re.match(lines[opening_index].rstrip())) + ) + if opening_index == 0 and len(lines) > 1: + displaced = lines[1].startswith("#!") and bool(lines[1][2:].strip()) + opening_ok = opening_ok and not displaced + last_ok = False + for raw in reversed(lines): + if raw.strip() == "": + continue + last_ok = bool(line_re.match(raw.rstrip())) + break + return (opening_ok, last_ok) +# ratios: loc_comments=161:57 imports_exports=4:7 calls_definitions=55:10 \ No newline at end of file From d307e28724e9efc885b54692776b89aba1b6649d Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 06:01:06 -0700 Subject: [PATCH 21/39] fix evidence packaging and byte hashing --- src/pubskill_lib/evidence.py | 75 +++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index 628ad55..68a0db8 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -1,43 +1,36 @@ """Evidence engine: inventory actual code before describing it. -Language comment markers and entry grammar are loaded from the vendored -canonical msdmd parser; this module does not maintain a second dialect. -``sha256`` is the stable source evidence hash: generated examiner NARRATIVE -blocks and RATIOS seals are excluded so the examiner cannot make its own -evidence stale. ``raw_sha256`` retains the literal file-content hash. +Language comment markers and entry grammar are loaded from the packaged copy of +the pinned canonical msdmd parser; this module does not maintain a second +dialect. ``sha256`` is the stable source evidence hash: generated examiner +NARRATIVE blocks and RATIOS seals are excluded when the source can be decoded. +``raw_sha256`` is always the literal file-byte hash. """ from __future__ import annotations import hashlib -import importlib.util +import io import re +import tokenize from dataclasses import dataclass, field -from functools import lru_cache from pathlib import Path +from . import _msdmd_universal as _canonical_msdmd from . import boundary SHEBANG_RE = re.compile(r"^#!.*$") _RATIOS_LINE_RE = re.compile(r"^(?:#|//|--|%|;|!|'|\*>)\s*ratios:\s*(.+?)\s*$") +_PYTHON_SUFFIXES = {".py", ".pyw", ".pyi"} -@lru_cache(maxsize=1) def _msdmd_parser(): - """Load the pinned repo-local canonical msdmd parser module.""" - repo = Path(__file__).resolve().parents[2] - parser_path = repo / ".agents" / "skills" / "msdmd" / "parsers" / "universal.py" - spec = importlib.util.spec_from_file_location("pubskill_lib._vendored_msdmd", parser_path) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load vendored msdmd parser: {parser_path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -@lru_cache(maxsize=1) + """Return the packaged, source-pinned canonical msdmd parser module.""" + return _canonical_msdmd + + def _comment_markers() -> dict[str, str]: - """Load COMMENT_MARKERS from the pinned repo-local msdmd parser.""" + """Load COMMENT_MARKERS from the packaged canonical msdmd parser.""" markers = getattr(_msdmd_parser(), "COMMENT_MARKERS", {}) return dict(markers) if isinstance(markers, dict) else {} @@ -98,6 +91,20 @@ def _block_names(text: str, marker: str) -> list[str]: return list(dict.fromkeys(match.group("name") for match in start_re.finditer(text))) +def _decode_source(path: Path, raw: bytes) -> tuple[str | None, str | None]: + """Decode source without changing byte identity; honor Python coding cookies.""" + if path.suffix.lower() in _PYTHON_SUFFIXES: + try: + encoding, _ = tokenize.detect_encoding(io.BytesIO(raw).readline) + return raw.decode(encoding), None + except (LookupError, SyntaxError, UnicodeDecodeError) as exc: + return None, f"source encoding unresolved: {exc}" + try: + return raw.decode("utf-8"), None + except UnicodeDecodeError as exc: + return None, f"source encoding unresolved: {exc}" + + def read_evidence(root: Path, path: Path) -> FileEvidence: root = Path(root).resolve() rel = str(path.relative_to(root)) @@ -105,29 +112,37 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: language = path.suffix.lower().lstrip(".") or "unknown" item = FileEvidence(path=rel, language=language, marker=marker) try: - text = path.read_text(encoding="utf-8", errors="replace") + raw = path.read_bytes() except OSError: item.hmmm.append("unreadable file") return item - raw_encoded = text.encode("utf-8", errors="replace") - stable_encoded = source_text(text, marker).encode("utf-8", errors="replace") - item.raw_sha256 = hashlib.sha256(raw_encoded).hexdigest() - item.sha256 = hashlib.sha256(stable_encoded).hexdigest() - item.size = len(raw_encoded) + item.raw_sha256 = hashlib.sha256(raw).hexdigest() + item.size = len(raw) try: item.executable = bool(path.stat().st_mode & 0o111) except OSError: pass + text, decode_hmmm = _decode_source(path, raw) + if text is None: + item.sha256 = item.raw_sha256 + item.marker = None + item.hmmm.append(decode_hmmm or "source encoding unresolved") + item.hmmm.append("metadata-excluding source hash unavailable; mutation disabled") + return item + + stable_encoded = source_text(text, marker).encode("utf-8") + item.sha256 = hashlib.sha256(stable_encoded).hexdigest() + first_line = text.splitlines()[0].rstrip() if text.splitlines() else "" if SHEBANG_RE.match(first_line): item.shebang = first_line if marker is not None: - for raw in text.splitlines(): - if _RATIOS_LINE_RE.match(raw.rstrip()): - item.ratios_lines.append(raw.rstrip()) + for raw_line in text.splitlines(): + if _RATIOS_LINE_RE.match(raw_line.rstrip()): + item.ratios_lines.append(raw_line.rstrip()) parser = _msdmd_parser() for name in _block_names(text, marker): entries = parser.parse_text(text, name, marker) From f550ae448e62a356ab4ccbcad07ef0953d2c3008 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 06:01:20 -0700 Subject: [PATCH 22/39] refresh fixture source pin --- examples/neglected-repo/expected-findings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/neglected-repo/expected-findings.json b/examples/neglected-repo/expected-findings.json index 223546c..40df805 100644 --- a/examples/neglected-repo/expected-findings.json +++ b/examples/neglected-repo/expected-findings.json @@ -1,7 +1,7 @@ { "schema_version": 1, "tool": "pubskill-lib", - "source_pin": "c14ee9d500579a4b5d6821f62c9d82ca96e73608", + "source_pin": "8de4f12d0f31ff94f41e4a0196c447c0cbe20faf", "target": { "path": "examples/neglected-repo", "commit": "hmmm", From 6fa9725636bcf39355ce13d18f39911b4dd62e6a Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 06:02:25 -0700 Subject: [PATCH 23/39] align apply test with safe adapter boundary --- tests/test_examine.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_examine.py b/tests/test_examine.py index e3f3d43..541e1f5 100644 --- a/tests/test_examine.py +++ b/tests/test_examine.py @@ -170,6 +170,7 @@ def test_dry_run_reports_without_writing(self): self.assertEqual(before, (self.root / "tool.py").read_text()) def test_apply_writes_ratios_and_assembles_docs(self): + shell_before = (self.root / "run.sh").read_bytes() result = self._run("--apply", "--out", "docs/examiner") self.assertEqual(0, result.returncode, result.stderr) @@ -180,7 +181,8 @@ def test_apply_writes_ratios_and_assembles_docs(self): shell = (self.root / "run.sh").read_text().splitlines() self.assertTrue(shell[0].startswith("#!")) - self.assertTrue(shell[1].startswith("# ratios: loc_comments=hmmm")) + self.assertEqual(shell_before, (self.root / "run.sh").read_bytes()) + self.assertFalse(any("ratios:" in line for line in shell)) ts = (self.root / "lib" / "util.ts").read_text().splitlines() self.assertTrue(ts[0].startswith("// ratios: loc_comments=")) @@ -201,4 +203,4 @@ def test_apply_writes_ratios_and_assembles_docs(self): if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file From 41ffc948a74a5872adc5d819d4917d60a048b361 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 06:03:31 -0700 Subject: [PATCH 24/39] add regressions for audit and evidence boundaries --- tests/test_repairs.py | 51 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/test_repairs.py b/tests/test_repairs.py index e5027b6..728ca39 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -1,5 +1,6 @@ """Regression tests for audit findings repaired on 2026-09-10.""" +import hashlib import json import os import tempfile @@ -125,6 +126,29 @@ def test_evidence_uses_vendored_msdmd_registry(self): self.assertEqual("//", markers[".ts"]) self.assertIn(".ps1", markers) + def test_raw_sha256_hashes_literal_python_bytes_and_honors_cookie(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "latin.py" + raw = b"# -*- coding: latin-1 -*-\nname = 'caf\xe9'\n" + path.write_bytes(raw) + item = evidence.read_evidence(root, path) + self.assertEqual(hashlib.sha256(raw).hexdigest(), item.raw_sha256) + self.assertEqual(len(raw), item.size) + self.assertEqual("#", item.marker) + self.assertFalse(item.hmmm) + + def test_undecodable_non_python_source_is_hmmm_and_not_mutable(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "bad.js" + raw = b"// invalid utf8: \xff\n" + path.write_bytes(raw) + item = evidence.read_evidence(root, path) + self.assertEqual(hashlib.sha256(raw).hexdigest(), item.raw_sha256) + self.assertIsNone(item.marker) + self.assertTrue(any("encoding unresolved" in text for text in item.hmmm)) + class PackageScriptTests(unittest.TestCase): def test_missing_local_package_script_is_a_defect(self): @@ -168,6 +192,31 @@ def test_interpreter_flags_do_not_hide_missing_local_script(self): self.assertTrue(any("missing.py" in claim for claim in claims)) self.assertTrue(any("missing.sh" in claim for claim in claims)) + def test_interpreter_non_file_modes_do_not_invent_script_paths(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "package.json").write_text( + json.dumps({"scripts": { + "node": "node -e console.log('ok')", + "python": "python -m http.server", + "shell": "bash -c 'echo ok'", + }}), + encoding="utf-8", + ) + document = audit.audit_path(root, "pin") + self.assertFalse([f for f in document["findings"] if f["surface"] == "deps"]) + + def test_absolute_local_package_script_reports_escape(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "package.json").write_text( + json.dumps({"scripts": {"build": "node /opt/project/build.js"}}), + encoding="utf-8", + ) + document = audit.audit_path(root, "pin") + claims = [f["claim"] for f in document["findings"] if f["surface"] == "deps"] + self.assertTrue(any("escapes repository via /opt/project/build.js" in claim for claim in claims)) + def test_non_object_package_manifest_is_target_defect(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -178,4 +227,4 @@ def test_non_object_package_manifest_is_target_defect(self): if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file From ea779edfe5e6d6094ee94eda5829184f58bfe570 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 06:03:52 -0700 Subject: [PATCH 25/39] guard packaged canonical parser provenance --- tests/test_provenance.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_provenance.py b/tests/test_provenance.py index f1aebac..79202c7 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -31,6 +31,11 @@ def test_fixture_source_pin_matches_publication_pin(self): ) self.assertEqual(_source_pin(), expected["source_pin"]) + def test_packaged_parser_matches_vendored_canonical_bytes(self): + vendored = REPO / ".agents" / "skills" / "msdmd" / "parsers" / "universal.py" + packaged = REPO / "src" / "pubskill_lib" / "_msdmd_universal.py" + self.assertEqual(vendored.read_bytes(), packaged.read_bytes()) + def test_local_secret_files_are_ignored(self): ignore = (REPO / ".gitignore").read_text(encoding="utf-8").splitlines() self.assertIn(".env", ignore) From 1394efe8796a1f07cb1c8a2eb30902b66ceb88d7 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Fri, 11 Sep 2026 06:04:13 -0700 Subject: [PATCH 26/39] verify noneditable wheel parser packaging --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b24b587..5ad6459 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,3 +22,8 @@ jobs: python -m pip install -e . python -m unittest discover -s tests python -m pubskill_lib.audit examples/neglected-repo --out /tmp/findings.json + python -m pip wheel . --no-deps -w /tmp/pubskill-wheel + python -m venv /tmp/pubskill-wheel-venv + /tmp/pubskill-wheel-venv/bin/python -m pip install --no-deps /tmp/pubskill-wheel/pubskill_lib-*.whl + cd /tmp + /tmp/pubskill-wheel-venv/bin/python -c "from pubskill_lib import evidence; assert evidence._comment_markers()['.py'] == '#'" From 3c2f989f8b8005a1e41f2589f9cd86e799cfdca9 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 01:42:44 +0000 Subject: [PATCH 27/39] Preserve examiner source encodings and resolve interpreter operands --- README.md | 10 +++++ src/pubskill_lib/audit.py | 52 ++++++++++++++++++++-- src/pubskill_lib/evidence.py | 13 +++--- src/pubskill_lib/examine.py | 52 +++++++++++++--------- src/pubskill_lib/msdmd_writer.py | 7 +-- tests/test_repairs.py | 74 ++++++++++++++++++++++++++++++++ 6 files changed, 175 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 050c044..84e78d2 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ It writes: The inspector does **not** yet clone URLs, select remote commits, execute target tests, or repair the target. Those are later capabilities and must not be inferred from the schema. +Direct script inspection handles interpreter flags and their arguments, such as +`python -W ignore app.py`, `node --require preload.js app.js`, and +`bash -o errexit build.sh`. Shell expansion and indirect launcher commands remain +outside this static inspection contract. + ## Repository examiner (BYOK) A separate documentation examiner builds on the repository evidence substrate: @@ -71,6 +76,11 @@ OPENAI_BASE_URL / ANTHROPIC_BASE_URL # process environment only Multiple configured providers are attempted sequentially as fallback. Unsupported or not-faithfully-computable metrics remain `hmmm`; they are not guessed. +Python coding cookies and UTF-8 byte-order marks are preserved during source +mutation. If generated prose cannot be encoded in the source encoding, the file +is left intact and the apply report records `hmmm`. Existing narratives remain +available in assembled documentation even when a file has no safe mutation adapter. + ## License MPL-2.0, same as skill-lib. Changes to MPL-covered files must be published. diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 7978965..90b09ea 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -35,6 +35,24 @@ "bash": {"-c"}, "sh": {"-c"}, } +VALUE_OPTIONS = { + "python": {"-W", "-X", "--check-hash-based-pycs"}, + "python3": {"-W", "-X", "--check-hash-based-pycs"}, + "bash": {"-o", "+o", "-O", "+O", "--rcfile", "--init-file"}, + "sh": {"-o", "+o"}, + "node": { + "-r", "--require", "--import", "--loader", "--experimental-loader", + "--conditions", "-C", "--input-type", "--env-file", + "--env-file-if-exists", "--inspect-port", "--inspect-publish-uid", + "--title", "--icu-data-dir", "--openssl-config", "--redirect-warnings", + "--trace-event-categories", "--trace-event-file-pattern", + "--unhandled-rejections", "--diagnostic-dir", "--report-directory", + "--report-filename", "--test-reporter", "--test-reporter-destination", + "--test-name-pattern", "--test-skip-pattern", "--test-concurrency", + "--test-shard", "--test-timeout", "--max-old-space-size", + "--stack-trace-limit", + }, +} class _Sink: @@ -168,7 +186,12 @@ def _check_pyproject_scripts(target, sink): def _local_script_targets(command): - """Yield direct local script operands without confusing interpreter modes with flags.""" + """Yield direct file operands after documented interpreter options. + + This is a static audit of direct invocations, not a shell evaluator. + Python -W/-X, Bash -o/-O and startup files, and common Node value options + consume their arguments; attached values and -- delimiters are supported. + """ for segment in re.split(r"\s*(?:&&|;|\|)\s*", command): if not segment.strip(): continue @@ -183,9 +206,32 @@ def _local_script_targets(command): index = 1 while index < len(tokens): token = tokens[index] - if token in non_file_modes: + if token == "--": + if index + 1 < len(tokens) and tokens[index + 1] != "-": + yield tokens[index + 1] break - if token.startswith("-"): + if token == "-" or token.split("=", 1)[0] in non_file_modes: + break + if token in VALUE_OPTIONS[interpreter]: + index += 2 + continue + if token.startswith("-") or (interpreter in {"bash", "sh"} and token.startswith("+")): + # Short options may be clustered or carry an attached argument. + non_file = False + if not token.startswith("--"): + modes = {mode[1:] for mode in non_file_modes if len(mode) == 2} + if interpreter in {"bash", "sh"}: + modes.add("s") # Read commands from stdin. + for position, option in enumerate(token[1:], start=1): + if token[0] == "-" and option in modes: + non_file = True + break + if token[0] + option in VALUE_OPTIONS[interpreter]: + if position == len(token) - 1: + index += 1 + break + if non_file: + break index += 1 continue yield token diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index 68a0db8..bb65bc5 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -81,6 +81,7 @@ class FileEvidence: raw_sha256: str = "" size: int = 0 executable: bool = False + encoding: str | None = "utf-8" hmmm: list[str] = field(default_factory=list) @@ -91,18 +92,18 @@ def _block_names(text: str, marker: str) -> list[str]: return list(dict.fromkeys(match.group("name") for match in start_re.finditer(text))) -def _decode_source(path: Path, raw: bytes) -> tuple[str | None, str | None]: +def _decode_source(path: Path, raw: bytes) -> tuple[str | None, str | None, str | None]: """Decode source without changing byte identity; honor Python coding cookies.""" if path.suffix.lower() in _PYTHON_SUFFIXES: try: encoding, _ = tokenize.detect_encoding(io.BytesIO(raw).readline) - return raw.decode(encoding), None + return raw.decode(encoding), encoding, None except (LookupError, SyntaxError, UnicodeDecodeError) as exc: - return None, f"source encoding unresolved: {exc}" + return None, None, f"source encoding unresolved: {exc}" try: - return raw.decode("utf-8"), None + return raw.decode("utf-8"), "utf-8", None except UnicodeDecodeError as exc: - return None, f"source encoding unresolved: {exc}" + return None, None, f"source encoding unresolved: {exc}" def read_evidence(root: Path, path: Path) -> FileEvidence: @@ -124,7 +125,7 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: except OSError: pass - text, decode_hmmm = _decode_source(path, raw) + text, item.encoding, decode_hmmm = _decode_source(path, raw) if text is None: item.sha256 = item.raw_sha256 item.marker = None diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index 57032ef..f782671 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -11,6 +11,7 @@ from __future__ import annotations import argparse +import hashlib import json import sys from datetime import datetime, timezone @@ -25,13 +26,6 @@ from . import ratios -def _read_text(path: Path) -> str: - try: - return path.read_text(encoding="utf-8", errors="replace") - except OSError: - return "" - - def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: supported = [ev for ev in evidence_list if ev.marker is not None] return { @@ -54,17 +48,30 @@ def _apply( ) -> tuple[dict, dict]: narratives: dict[str, dict[str, str]] = {} changed: list[str] = [] + unresolved: dict[str, str] = {} now = datetime.now(timezone.utc).isoformat() engine = ratios.RatiosEngine() for ev in evidence_list: path = boundary.assert_inside(root, root / ev.path) - original_text = _read_text(path) + if ev.narrative_entries: + narratives[ev.path] = ev.narrative_entries[0] adapter = engine.adapter_for(path) - if ev.marker is None or adapter is None: + if ev.marker is None or adapter is None or ev.encoding is None: + continue + try: + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != ev.raw_sha256: + unresolved[ev.path] = "source changed after inventory; mutation skipped" + continue + original_text = raw.decode(ev.encoding) + except (OSError, UnicodeError) as exc: + unresolved[ev.path] = f"source unavailable; mutation skipped: {exc}" continue new_text = original_text + file_changes: list[str] = [] + entry = narratives.get(ev.path) if narrate: result = narrative.narrate_file( ev, @@ -72,31 +79,32 @@ def _apply( provider_list, now, ) - narratives[ev.path] = result.entry + entry = result.entry if result.hmmm: result.entry["summary"] = result.entry["summary"] or "hmmm" - else: - existing = ev.narrative_entries[0] if ev.narrative_entries else None - if existing: - narratives[ev.path] = existing - - entry = narratives.get(ev.path) if entry: new_text, block_changed = msdmd_writer.upsert_narrative( new_text, ev.marker, entry, path ) if block_changed: - changed.append(f"{ev.path}:narrative") + file_changes.append(f"{ev.path}:narrative") values = engine.compute(path, evidence.source_text(new_text, ev.marker)) new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) if ratio_changed: - changed.append(f"{ev.path}:ratios") + file_changes.append(f"{ev.path}:ratios") if new_text != original_text: - msdmd_writer.write_text_safely(path, new_text) + try: + msdmd_writer.write_text_safely(path, new_text, ev.encoding) + except UnicodeEncodeError: + unresolved[ev.path] = f"generated text cannot use {ev.encoding}; mutation skipped" + continue + if entry: + narratives[ev.path] = entry + changed.extend(file_changes) - return narratives, {"changed": changed, "narrated": len(narratives)} + return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved} def main(argv: list[str] | None = None) -> int: @@ -140,11 +148,13 @@ def main(argv: list[str] | None = None) -> int: volume = assemble.assemble_docs(root, evidence_list, narratives, out_dir) if args.json: - print(json.dumps({"changed": report["changed"], "volume": str(volume)}, indent=2)) + print(json.dumps({"changed": report["changed"], "hmmm": report["hmmm"], "volume": str(volume)}, indent=2)) else: print(f"applied: {len(report['changed'])} writes") for change in report["changed"]: print(f" {change}") + for path, reason in report["hmmm"].items(): + print(f" hmmm: {path}: {reason}") print(f"assembled: {volume}") return 0 diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 491b240..59df970 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -71,13 +71,14 @@ def upsert_narrative( return new_text, new_text != text -def write_text_safely(path: Path, new_text: str) -> None: - """Write text without changing the file's executable bit.""" +def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8") -> None: + """Preserve the source encoding and mode; encode before opening for writing.""" + encoded = new_text.encode(encoding) mode = None try: mode = path.stat().st_mode & 0o777 except OSError: pass - path.write_text(new_text, encoding="utf-8") + path.write_bytes(encoded) if mode is not None: path.chmod(mode) diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 728ca39..5a86f06 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -119,6 +119,61 @@ def test_apply_skips_parser_supported_language_without_safe_ratio_adapter(self): self.assertEqual([], report["changed"]) + def test_apply_preserves_encoding_and_source_identity(self): + class FakeProvider: + name, model = "fake", "model-1" + def chat(self, system, user): + assert "café" in user + return "Prints café." + + for encoding in ("latin-1", "utf-8-sig"): + with self.subTest(encoding=encoding), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "tool.py" + text = "# coding: " + ("utf-8" if encoding == "utf-8-sig" else encoding) + "\nprint('café')\n" + path.write_bytes(text.encode(encoding)) + before = evidence.read_evidence(root, path) + examine._apply(root, [before], [FakeProvider()], True) + after = evidence.read_evidence(root, path) + self.assertEqual(before.sha256, after.sha256) + self.assertFalse(narrative.is_stale(after.narrative_entries[0], after.sha256)) + self.assertIn("café", path.read_bytes().decode(encoding)) + compile(path.read_bytes(), str(path), "exec") + first = path.read_bytes() + _, report = examine._apply(root, [after], [], False) + self.assertEqual(first, path.read_bytes()) + self.assertEqual([], report["changed"]) + + def test_unrepresentable_narrative_does_not_truncate_source(self): + class FakeProvider: + name, model = "fake", "model-1" + def chat(self, system, user): + return "A snowman: \u2603" + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "tool.py" + raw = b"# coding: latin-1\nprint('caf\xe9')\n" + path.write_bytes(raw) + narratives, report = examine._apply(root, [evidence.read_evidence(root, path)], [FakeProvider()], True) + self.assertEqual(raw, path.read_bytes()) + self.assertEqual({}, narratives) + self.assertEqual([], report["changed"]) + self.assertIn("tool.py", report["hmmm"]) + + def test_adapterless_narratives_are_retained_without_writes(self): + for filename, marker in (("tool.sh", "#"), ("index.php", "//")): + with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / filename + original = "\n".join((f"{marker} === NARRATIVE ===", f"{marker} id: existing", f"{marker} summary: Retained explanation.", f"{marker} === END NARRATIVE ===", "")) + path.write_text(original) + result, report = examine._apply(root, [evidence.read_evidence(root, path)], [], False) + self.assertEqual("Retained explanation.", result[filename]["summary"]) + self.assertEqual(original, path.read_text()) + self.assertEqual([], report["changed"]) + + class CanonicalMarkerTests(unittest.TestCase): def test_evidence_uses_vendored_msdmd_registry(self): markers = evidence._comment_markers() @@ -217,6 +272,25 @@ def test_absolute_local_package_script_reports_escape(self): claims = [f["claim"] for f in document["findings"] if f["surface"] == "deps"] self.assertTrue(any("escapes repository via /opt/project/build.js" in claim for claim in claims)) + def test_value_taking_interpreter_options_select_actual_file(self): + commands = ( + "python -W ignore app.py", "python3 -X dev app.py", + "python -uW ignore app.py", "python -Wignore app.py", + "python --check-hash-based-pycs always app.py", + "node --require preload.js app.py", "node -rpreload.js app.py", + "node --import preload.js --trace-warnings app.py", + "node --max-old-space-size 512 app.py", + "bash -o errexit app.py", "bash -O extglob app.py", + "bash --rcfile startup.sh app.py", "bash -eo pipefail app.py", + "sh +o errexit app.py", "python -- app.py", + ) + for command in commands: + with self.subTest(command=command): + self.assertEqual(["app.py"], list(audit._local_script_targets(command))) + for command in ("python -W ignore -c pass", "python -mhttp.server", "node --eval=1", "bash -ec 'echo ok'", "sh -s arg", "python - arg", "node -r preload.js -e 1", "python -W"): + with self.subTest(command=command): + self.assertEqual([], list(audit._local_script_targets(command))) + def test_non_object_package_manifest_is_target_defect(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From b35b33ac94a262ac6c78188b8b04cb61de31c77c Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 01:57:29 +0000 Subject: [PATCH 28/39] Preserve BOMs across adapters and complete Node option arity --- src/pubskill_lib/audit.py | 33 ++++++++++++++++++++++----------- src/pubskill_lib/evidence.py | 3 ++- tests/test_repairs.py | 18 +++++++++++++++++- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 90b09ea..e3c49b8 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -29,7 +29,7 @@ PIN_PATTERN = re.compile(r"`([0-9a-f]{40})`") LOCAL_SCRIPT_INTERPRETERS = {"node", "python", "python3", "bash", "sh"} NON_FILE_MODES = { - "node": {"-e", "--eval", "-p", "--print"}, + "node": {"-e", "--eval", "-p", "--print", "--run"}, "python": {"-c", "-m"}, "python3": {"-c", "-m"}, "bash": {"-c"}, @@ -41,16 +41,27 @@ "bash": {"-o", "+o", "-O", "+O", "--rcfile", "--init-file"}, "sh": {"-o", "+o"}, "node": { - "-r", "--require", "--import", "--loader", "--experimental-loader", - "--conditions", "-C", "--input-type", "--env-file", - "--env-file-if-exists", "--inspect-port", "--inspect-publish-uid", - "--title", "--icu-data-dir", "--openssl-config", "--redirect-warnings", - "--trace-event-categories", "--trace-event-file-pattern", - "--unhandled-rejections", "--diagnostic-dir", "--report-directory", - "--report-filename", "--test-reporter", "--test-reporter-destination", - "--test-name-pattern", "--test-skip-pattern", "--test-concurrency", - "--test-shard", "--test-timeout", "--max-old-space-size", - "--stack-trace-limit", + "--allow-fs-read", "--allow-fs-write", "--build-snapshot-config", "--conditions", + "--cpu-prof-dir", "--cpu-prof-interval", "--cpu-prof-name", "--debug-port", + "--diagnostic-dir", "--disable-proto", "--disable-warning", "--dns-result-order", + "--env-file", "--env-file-if-exists", "--experimental-config-file", + "--experimental-default-type", "--experimental-loader", "--experimental-sea-config", + "--experimental-test-isolation", "--heap-prof-dir", "--heap-prof-interval", + "--heap-prof-name", "--heapsnapshot-near-heap-limit", "--heapsnapshot-signal", + "--icu-data-dir", "--import", "--input-type", "--inspect-port", + "--inspect-publish-uid", "--loader", "--localstorage-file", "--max-http-header-size", + "--max-old-space-size", "--max-old-space-size-percentage", + "--network-family-autoselection-attempt-timeout", "--openssl-config", + "--redirect-warnings", "--report-dir", "--report-directory", "--report-filename", + "--report-signal", "--require", "--secure-heap", "--secure-heap-min", + "--snapshot-blob", "--stack-trace-limit", "--test-concurrency", + "--test-coverage-branches", "--test-coverage-exclude", "--test-coverage-functions", + "--test-coverage-include", "--test-coverage-lines", "--test-name-pattern", + "--test-reporter", "--test-reporter-destination", "--test-shard", + "--test-skip-pattern", "--test-timeout", "--title", "--tls-cipher-list", + "--tls-keylog", "--trace-event-categories", "--trace-event-file-pattern", + "--trace-require-module", "--unhandled-rejections", "--use-largepages", + "--v8-pool-size", "--watch-kill-signal", "--watch-path", "-C", "-r", }, } diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index bb65bc5..5bb6e09 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -101,7 +101,8 @@ def _decode_source(path: Path, raw: bytes) -> tuple[str | None, str | None, str except (LookupError, SyntaxError, UnicodeDecodeError) as exc: return None, None, f"source encoding unresolved: {exc}" try: - return raw.decode("utf-8"), "utf-8", None + encoding = "utf-8-sig" if raw.startswith(b"\xef\xbb\xbf") else "utf-8" + return raw.decode(encoding), encoding, None except UnicodeDecodeError as exc: return None, None, f"source encoding unresolved: {exc}" diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 5a86f06..9cf3726 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -161,6 +161,21 @@ def chat(self, system, user): self.assertEqual([], report["changed"]) self.assertIn("tool.py", report["hmmm"]) + def test_non_python_bom_stays_at_byte_zero(self): + for name, body in (("main.c", "int main(void) { return 0; }\n"), ("main.rs", "fn main() {}\n")): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / name + path.write_bytes(body.encode("utf-8-sig")) + before = evidence.read_evidence(root, path) + examine._apply(root, [before], [], False) + raw = path.read_bytes() + self.assertTrue(raw.startswith(b"\xef\xbb\xbf")) + self.assertEqual(1, raw.count(b"\xef\xbb\xbf")) + self.assertEqual(before.sha256, evidence.read_evidence(root, path).sha256) + examine._apply(root, [evidence.read_evidence(root, path)], [], False) + self.assertEqual(raw, path.read_bytes()) + def test_adapterless_narratives_are_retained_without_writes(self): for filename, marker in (("tool.sh", "#"), ("index.php", "//")): with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp: @@ -280,6 +295,7 @@ def test_value_taking_interpreter_options_select_actual_file(self): "node --require preload.js app.py", "node -rpreload.js app.py", "node --import preload.js --trace-warnings app.py", "node --max-old-space-size 512 app.py", + "node --watch --watch-path src app.py", "bash -o errexit app.py", "bash -O extglob app.py", "bash --rcfile startup.sh app.py", "bash -eo pipefail app.py", "sh +o errexit app.py", "python -- app.py", @@ -301,4 +317,4 @@ def test_non_object_package_manifest_is_target_defect(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 45ecb3fe4d96708af38bf9a9f1e1f9ebb8be31e4 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 02:13:22 +0000 Subject: [PATCH 29/39] Handle Node test runner value options and aliases --- src/pubskill_lib/audit.py | 5 +++-- tests/test_repairs.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index e3c49b8..8677155 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -46,17 +46,18 @@ "--diagnostic-dir", "--disable-proto", "--disable-warning", "--dns-result-order", "--env-file", "--env-file-if-exists", "--experimental-config-file", "--experimental-default-type", "--experimental-loader", "--experimental-sea-config", - "--experimental-test-isolation", "--heap-prof-dir", "--heap-prof-interval", + "--experimental-package-map", "--experimental-test-tag-filter", "--experimental-test-isolation", "--heap-prof-dir", "--heap-prof-interval", "--heap-prof-name", "--heapsnapshot-near-heap-limit", "--heapsnapshot-signal", "--icu-data-dir", "--import", "--input-type", "--inspect-port", "--inspect-publish-uid", "--loader", "--localstorage-file", "--max-http-header-size", - "--max-old-space-size", "--max-old-space-size-percentage", + "--max-old-space-size", "--max-old-space-size-percentage", "--max-semi-space-size", "--network-family-autoselection-attempt-timeout", "--openssl-config", "--redirect-warnings", "--report-dir", "--report-directory", "--report-filename", "--report-signal", "--require", "--secure-heap", "--secure-heap-min", "--snapshot-blob", "--stack-trace-limit", "--test-concurrency", "--test-coverage-branches", "--test-coverage-exclude", "--test-coverage-functions", "--test-coverage-include", "--test-coverage-lines", "--test-name-pattern", + "--test-global-setup", "--test-isolation", "--test-random-seed", "--test-rerun-failures", "--test-reporter", "--test-reporter-destination", "--test-shard", "--test-skip-pattern", "--test-timeout", "--title", "--tls-cipher-list", "--tls-keylog", "--trace-event-categories", "--trace-event-file-pattern", diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 9cf3726..cd359bf 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -296,6 +296,12 @@ def test_value_taking_interpreter_options_select_actual_file(self): "node --import preload.js --trace-warnings app.py", "node --max-old-space-size 512 app.py", "node --watch --watch-path src app.py", + "node --test-isolation none app.py", + "node --experimental-test-isolation none app.py", + "node --test-global-setup setup.js app.py", + "node --test-rerun-failures failures.json app.py", + "node --test-random-seed 12 app.py", + "node --max-semi-space-size 16 app.py", "bash -o errexit app.py", "bash -O extglob app.py", "bash --rcfile startup.sh app.py", "bash -eo pipefail app.py", "sh +o errexit app.py", "python -- app.py", From 51c7ef16c7ceefaf2dd40821dbc18e8da33570ea Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 02:35:20 +0000 Subject: [PATCH 30/39] Preserve shell quoting, URL entrypoints, and canonical parser bytes --- src/pubskill_lib/audit.py | 49 ++++++++++++++++++++++++++++++++++--- src/pubskill_lib/examine.py | 13 ++++++++-- tests/test_repairs.py | 40 ++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 8677155..4da9f74 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -11,6 +11,7 @@ import json import re import shlex +from urllib.parse import unquote, urlsplit import subprocess import sys from pathlib import Path @@ -197,6 +198,39 @@ def _check_pyproject_scripts(target, sink): ) +def _shell_segments(command): + """Split direct shell commands while retaining quoted/escaped separators.""" + start, quote, escaped = 0, None, False + for index, character in enumerate(command): + if escaped: + escaped = False + elif character == "\\" and quote != "'": + escaped = True + elif quote: + if character == quote: + quote = None + elif character in {"'", '"'}: + quote = character + elif character in ";&|\n": + yield command[start:index] + start = index + 1 + yield command[start:] + + +def _entrypoint_target(token, entry_url): + if not entry_url: + return token + try: + parsed = urlsplit(token) + if parsed.scheme == "file" and parsed.netloc in {"", "localhost"}: + return unquote(parsed.path, errors="strict") + if not parsed.scheme: + return unquote(parsed.path, errors="strict") # Relative entry URL. + except (ValueError, UnicodeError): + pass + return None + + def _local_script_targets(command): """Yield direct file operands after documented interpreter options. @@ -204,7 +238,7 @@ def _local_script_targets(command): Python -W/-X, Bash -o/-O and startup files, and common Node value options consume their arguments; attached values and -- delimiters are supported. """ - for segment in re.split(r"\s*(?:&&|;|\|)\s*", command): + for segment in _shell_segments(command): if not segment.strip(): continue try: @@ -215,12 +249,19 @@ def _local_script_targets(command): continue interpreter = tokens[0] non_file_modes = NON_FILE_MODES[interpreter] + entry_url = False index = 1 while index < len(tokens): token = tokens[index] + if interpreter == "node" and token in {"--entry-url", "--experimental-entry-url"}: + entry_url = True + index += 1 + continue if token == "--": if index + 1 < len(tokens) and tokens[index + 1] != "-": - yield tokens[index + 1] + target = _entrypoint_target(tokens[index + 1], entry_url) + if target is not None: + yield target break if token == "-" or token.split("=", 1)[0] in non_file_modes: break @@ -246,7 +287,9 @@ def _local_script_targets(command): break index += 1 continue - yield token + target = _entrypoint_target(token, entry_url) + if target is not None: + yield target break diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index f782671..ac3ff21 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -26,12 +26,17 @@ from . import ratios +def _canonical_artifact(path: Path) -> bool: + return path.name == "_msdmd_universal.py" and path.parent.name == "pubskill_lib" + + def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: - supported = [ev for ev in evidence_list if ev.marker is not None] + supported = [ev for ev in evidence_list if ev.marker is not None and not _canonical_artifact(Path(ev.path))] return { "root": str(root), "files": len(evidence_list), "supported_files": len(supported), + "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(Path(ev.path))], "unsupported": [ {"path": ev.path, "hmmm": ev.hmmm} for ev in evidence_list if ev.marker is None ], @@ -49,6 +54,7 @@ def _apply( narratives: dict[str, dict[str, str]] = {} changed: list[str] = [] unresolved: dict[str, str] = {} + preserved_authority: list[str] = [] now = datetime.now(timezone.utc).isoformat() engine = ratios.RatiosEngine() @@ -56,6 +62,9 @@ def _apply( path = boundary.assert_inside(root, root / ev.path) if ev.narrative_entries: narratives[ev.path] = ev.narrative_entries[0] + if _canonical_artifact(path): + preserved_authority.append(ev.path) + continue adapter = engine.adapter_for(path) if ev.marker is None or adapter is None or ev.encoding is None: continue @@ -104,7 +113,7 @@ def _apply( narratives[ev.path] = entry changed.extend(file_changes) - return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved} + return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved, "preserved_authority": preserved_authority} def main(argv: list[str] | None = None) -> int: diff --git a/tests/test_repairs.py b/tests/test_repairs.py index cd359bf..04f7464 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -47,6 +47,20 @@ def test_blank_model_override_uses_provider_default(self): class NarrativeBoundaryTests(unittest.TestCase): + def test_apply_preserves_packaged_canonical_parser(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + parser = root / "src/pubskill_lib/_msdmd_universal.py" + parser.parent.mkdir(parents=True) + original = Path(evidence._canonical_msdmd.__file__).read_bytes() + parser.write_bytes(original) + ev = evidence.read_evidence(root, parser) + _, report = examine._apply(root, [ev], [], True) + self.assertEqual(original, parser.read_bytes()) + self.assertEqual([], report["changed"]) + self.assertEqual([ev.path], report["preserved_authority"]) + self.assertEqual(0, examine._plan(root, [ev])["supported_files"]) + def test_narrative_preserves_python_shebang_and_encoding_header(self): text = ( "#!/usr/bin/env python3\n" @@ -302,6 +316,11 @@ def test_value_taking_interpreter_options_select_actual_file(self): "node --test-rerun-failures failures.json app.py", "node --test-random-seed 12 app.py", "node --max-semi-space-size 16 app.py", + "node --test-name-pattern 'unit|integration' app.py", + "node --test-name-pattern 'unit;integration' app.py", + "node --test-name-pattern '|' app.py", + "node --test-name-pattern unit\\|integration app.py", + "node --inspect=9229 app.py", "node --inspect app.py", "bash -o errexit app.py", "bash -O extglob app.py", "bash --rcfile startup.sh app.py", "bash -eo pipefail app.py", "sh +o errexit app.py", "python -- app.py", @@ -313,6 +332,27 @@ def test_value_taking_interpreter_options_select_actual_file(self): with self.subTest(command=command): self.assertEqual([], list(audit._local_script_targets(command))) + def test_inspector_endpoint_requires_equals_in_node_24(self): + # Official Node v24.15.0 attempts to load 9229 as the entry file here. + self.assertEqual(["9229"], list(audit._local_script_targets("node --inspect 9229 app.js"))) + + def test_quoted_segments_and_entrypoint_urls(self): + self.assertEqual(["first.js", "second.js"], list(audit._local_script_targets("node --test-name-pattern 'a|b' first.js && node second.js"))) + self.assertEqual([], list(audit._local_script_targets("node --entry-url 'data:text/javascript,console.log(1);'"))) + self.assertEqual(["/definitely/missing file.js"], list(audit._local_script_targets("node --entry-url file:///definitely/missing%20file.js"))) + self.assertEqual(["./local file.js"], list(audit._local_script_targets("node --entry-url './local%20file.js?debug=1#part'"))) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "package.json").write_text(json.dumps({"scripts": { + "file": "node --entry-url file:///definitely/missing.js", + "data": "node --entry-url 'data:text/javascript,console.log(1);'", + "quoted": "node --test-name-pattern 'unit|integration' missing.js", + }})) + claims = [f["claim"] for f in audit.audit_path(root, "pin")["findings"] if f["surface"] == "deps"] + self.assertEqual(2, len(claims)) + self.assertTrue(any("escapes repository via /definitely/missing.js" in claim for claim in claims)) + self.assertTrue(any("missing local file missing.js" in claim for claim in claims)) + def test_non_object_package_manifest_is_target_defect(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From 25f65b5b564a3b3a439de4ee938453f95da5892c Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 03:01:05 +0000 Subject: [PATCH 31/39] Preserve literal source data and concurrent edits during examination --- src/pubskill_lib/audit.py | 60 ++++++++++++++------- src/pubskill_lib/evidence.py | 35 +++++------- src/pubskill_lib/examine.py | 15 ++++-- src/pubskill_lib/msdmd_writer.py | 48 +++++++++-------- src/pubskill_lib/ratios.py | 19 +++---- src/pubskill_lib/source_boundaries.py | 40 ++++++++++++++ tests/test_repairs.py | 78 +++++++++++++++++++++++++++ 7 files changed, 218 insertions(+), 77 deletions(-) create mode 100644 src/pubskill_lib/source_boundaries.py diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 4da9f74..c529ae4 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -217,21 +217,26 @@ def _shell_segments(command): yield command[start:] -def _entrypoint_target(token, entry_url): - if not entry_url: - return token +def _entrypoint_target(token, entry_url, unresolved=None): try: - parsed = urlsplit(token) - if parsed.scheme == "file" and parsed.netloc in {"", "localhost"}: - return unquote(parsed.path, errors="strict") - if not parsed.scheme: - return unquote(parsed.path, errors="strict") # Relative entry URL. - except (ValueError, UnicodeError): - pass + target = token + if entry_url: + parsed = urlsplit(token) + if parsed.scheme == "file" and parsed.netloc not in {"", "localhost"}: + raise ValueError("unsupported file URL authority") + if parsed.scheme not in {"", "file"}: + return None + target = unquote(parsed.path, errors="strict") + if not target or "\0" in target: + raise ValueError("empty or NUL-containing path") + return target + except (ValueError, UnicodeError) as error: + if unresolved is not None: + unresolved.append(f"unresolved entrypoint {token!r}: {error}") return None -def _local_script_targets(command): +def _local_script_targets(command, unresolved=None): """Yield direct file operands after documented interpreter options. This is a static audit of direct invocations, not a shell evaluator. @@ -253,13 +258,26 @@ def _local_script_targets(command): index = 1 while index < len(tokens): token = tokens[index] + if interpreter == "node" and token == "inspect": + arguments = tokens[index + 1:] + if not arguments: + break + target = arguments[0] + if re.fullmatch(r"[^:]+:\d+", target) or (len(arguments) == 2 and target == "-p" and arguments[1].isdigit()): + break # Attach to an existing debugger/process, not a file. + if re.fullmatch(r"--port=\d+", target): + target = arguments[1] if len(arguments) > 1 else "" + target = _entrypoint_target(target, False, unresolved) + if target is not None: + yield target + break if interpreter == "node" and token in {"--entry-url", "--experimental-entry-url"}: entry_url = True index += 1 continue if token == "--": if index + 1 < len(tokens) and tokens[index + 1] != "-": - target = _entrypoint_target(tokens[index + 1], entry_url) + target = _entrypoint_target(tokens[index + 1], entry_url, unresolved) if target is not None: yield target break @@ -287,13 +305,13 @@ def _local_script_targets(command): break index += 1 continue - target = _entrypoint_target(token, entry_url) + target = _entrypoint_target(token, entry_url, unresolved) if target is not None: yield target break -def _check_package_scripts(target, sink): +def _check_package_scripts(target, sink, unresolved): package = target / "package.json" text = _read_text(package) if text is None: @@ -312,12 +330,17 @@ def _check_package_scripts(target, sink): for name, command in sorted(scripts.items()): if not isinstance(command, str): continue - for raw_path in _local_script_targets(command): + script_unresolved = [] + for raw_path in _local_script_targets(command, script_unresolved): raw_path = raw_path.strip('"\'') if "://" in raw_path: continue - candidate = Path(raw_path) - local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() + try: + candidate = Path(raw_path) + local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() + except (ValueError, OSError) as error: + script_unresolved.append(f"unresolved local path {raw_path!r}: {error}") + continue try: local.relative_to(target.resolve()) except ValueError: @@ -329,6 +352,7 @@ def _check_package_scripts(target, sink): f"package script {name} points to missing local file {raw_path}", "package.json [scripts]", ) + unresolved.extend(f"package script {name}: {item}" for item in script_unresolved) def _read_source_pin(): @@ -367,7 +391,7 @@ def audit_path(target_path, source_pin=None): if has_pyproject: _check_pyproject_scripts(target, sink) if has_package: - _check_package_scripts(target, sink) + _check_package_scripts(target, sink, document["hmmm"]) document["surfaces"] = surfaces document["findings"] = sink.finalize() diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index 5bb6e09..d0c51ff 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -18,6 +18,7 @@ from . import _msdmd_universal as _canonical_msdmd from . import boundary +from . import ratios, source_boundaries SHEBANG_RE = re.compile(r"^#!.*$") _RATIOS_LINE_RE = re.compile(r"^(?:#|//|--|%|;|!|'|\*>)\s*ratios:\s*(.+?)\s*$") @@ -35,7 +36,7 @@ def _comment_markers() -> dict[str, str]: return dict(markers) if isinstance(markers, dict) else {} -def source_text(text: str, marker: str | None) -> str: +def source_text(text: str, marker: str | None, path: Path | None = None) -> str: """Return source text with complete generated NARRATIVE/RATIOS metadata removed. Trailing blank lines are normalized because RATIOS placement already removes @@ -44,24 +45,11 @@ def source_text(text: str, marker: str | None) -> str: if marker is None: return text - start = f"{marker} === NARRATIVE ===" - end = f"{marker} === END NARRATIVE ===" lines = text.splitlines() - kept: list[str] = [] - index = 0 - - while index < len(lines): - raw = lines[index] - if raw.rstrip() == start: - close = index + 1 - while close < len(lines) and lines[close].rstrip() != end: - close += 1 - if close < len(lines): - index = close + 1 - continue - if not _RATIOS_LINE_RE.match(raw.rstrip()): - kept.append(raw) - index += 1 + adapter = ratios.default_adapter_for(path) if path is not None else None + bookends, narrative = source_boundaries.metadata_indices(lines, marker, adapter) + excluded = bookends | narrative + kept = [line for index, line in enumerate(lines) if index not in excluded] while kept and not kept[-1].strip(): kept.pop() @@ -134,7 +122,7 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: item.hmmm.append("metadata-excluding source hash unavailable; mutation disabled") return item - stable_encoded = source_text(text, marker).encode("utf-8") + stable_encoded = source_text(text, marker, path).encode("utf-8") item.sha256 = hashlib.sha256(stable_encoded).hexdigest() first_line = text.splitlines()[0].rstrip() if text.splitlines() else "" @@ -142,12 +130,13 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: item.shebang = first_line if marker is not None: - for raw_line in text.splitlines(): - if _RATIOS_LINE_RE.match(raw_line.rstrip()): - item.ratios_lines.append(raw_line.rstrip()) + lines = text.splitlines() + bookends, narrative_indices = source_boundaries.metadata_indices(lines, marker, ratios.default_adapter_for(path)) + item.ratios_lines = [lines[index].rstrip() for index in sorted(bookends)] parser = _msdmd_parser() for name in _block_names(text, marker): - entries = parser.parse_text(text, name, marker) + block_text = "\n".join(lines[index] for index in sorted(narrative_indices)) if name == "NARRATIVE" else text + entries = parser.parse_text(block_text, name, marker) item.msdmd_blocks[name] = entries item.narrative_entries = item.msdmd_blocks.get("NARRATIVE", []) else: diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index ac3ff21..1c943d1 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -84,7 +84,7 @@ def _apply( if narrate: result = narrative.narrate_file( ev, - evidence.source_text(original_text, ev.marker), + evidence.source_text(original_text, ev.marker, path), provider_list, now, ) @@ -98,14 +98,23 @@ def _apply( if block_changed: file_changes.append(f"{ev.path}:narrative") - values = engine.compute(path, evidence.source_text(new_text, ev.marker)) + values = engine.compute(path, evidence.source_text(new_text, ev.marker, path)) new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) if ratio_changed: file_changes.append(f"{ev.path}:ratios") if new_text != original_text: try: - msdmd_writer.write_text_safely(path, new_text, ev.encoding) + if path.is_symlink() or boundary.assert_inside(root, path) != path or path.read_bytes() != raw: + unresolved[ev.path] = "source changed during examination; mutation skipped" + continue + msdmd_writer.write_text_safely(path, new_text, ev.encoding, expected_raw=raw) + except msdmd_writer.SourceChangedError: + unresolved[ev.path] = "source changed during examination; mutation skipped" + continue + except OSError as exc: + unresolved[ev.path] = f"source unavailable before write; mutation skipped: {exc}" + continue except UnicodeEncodeError: unresolved[ev.path] = f"generated text cannot use {ev.encoding}; mutation skipped" continue diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 59df970..5192e33 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -8,8 +8,11 @@ from __future__ import annotations from pathlib import Path +import os +import tempfile from . import ratios +from . import source_boundaries NARRATIVE_BLOCK = "NARRATIVE" @@ -22,24 +25,11 @@ def _block_lines(marker: str, entry: dict[str, str]) -> list[str]: return lines -def _without_narrative_lines(text: str, marker: str) -> list[str]: +def _without_narrative_lines(text: str, marker: str, adapter=None) -> list[str]: """Remove complete NARRATIVE blocks without creating phantom blank lines.""" - start = f"{marker} === {NARRATIVE_BLOCK} ===" - end = f"{marker} === END {NARRATIVE_BLOCK} ===" lines = text.splitlines() - kept: list[str] = [] - index = 0 - - while index < len(lines): - if lines[index].rstrip() == start: - close = index + 1 - while close < len(lines) and lines[close].rstrip() != end: - close += 1 - if close < len(lines): - index = close + 1 - continue - kept.append(lines[index]) - index += 1 + _, indices = source_boundaries.metadata_indices(lines, marker, adapter) + kept = [line for index, line in enumerate(lines) if index not in indices] while kept and not kept[-1].strip(): kept.pop() @@ -57,10 +47,10 @@ def upsert_narrative( path: Path | None = None, ) -> tuple[str, bool]: """Replace NARRATIVE blocks without crossing the protected opening boundary.""" - lines = _without_narrative_lines(text, marker) + adapter = ratios.RatiosEngine().adapter_for(path) if path is not None else None + lines = _without_narrative_lines(text, marker, adapter) block = "\n".join(_block_lines(marker, entry)) - adapter = ratios.RatiosEngine().adapter_for(path) if path is not None else None insert_at = ratios.opening_index(lines, adapter) ratios_prefix = f"{marker} ratios:" if insert_at < len(lines) and lines[insert_at].lstrip().startswith(ratios_prefix): @@ -71,7 +61,11 @@ def upsert_narrative( return new_text, new_text != text -def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8") -> None: +class SourceChangedError(RuntimeError): + """The live source no longer matches the inventoried bytes.""" + + +def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, expected_raw: bytes | None = None) -> None: """Preserve the source encoding and mode; encode before opening for writing.""" encoded = new_text.encode(encoding) mode = None @@ -79,6 +73,16 @@ def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8") -> Non mode = path.stat().st_mode & 0o777 except OSError: pass - path.write_bytes(encoded) - if mode is not None: - path.chmod(mode) + temporary = None + try: + with tempfile.NamedTemporaryFile(mode="wb", dir=path.parent, prefix=".examiner-", delete=False) as stream: + temporary = Path(stream.name) + stream.write(encoded) + if mode is not None: + temporary.chmod(mode) + if expected_raw is not None and (path.is_symlink() or path.read_bytes() != expected_raw): + raise SourceChangedError("source changed before metadata publication") + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) diff --git a/src/pubskill_lib/ratios.py b/src/pubskill_lib/ratios.py index 7df538a..3d523c5 100644 --- a/src/pubskill_lib/ratios.py +++ b/src/pubskill_lib/ratios.py @@ -27,6 +27,7 @@ import re from pathlib import Path +from . import source_boundaries RATIO_IDS = ("loc_comments", "imports_exports", "calls_definitions") SHEBANG_RE = re.compile(r"^#!.*$") @@ -135,10 +136,11 @@ def render_ratios_line(marker: str, values: dict[str, str]) -> str: return f"{marker} ratios: {body}" -def strip_ratios_lines(text: str, marker: str) -> list[str]: - """Return the file's lines with every ratios line removed.""" - line_re = _ratios_line_re(marker) - return [line for line in text.splitlines() if not line_re.match(line.rstrip())] +def strip_ratios_lines(text: str, marker: str, adapter=None) -> list[str]: + """Remove only the reserved bookends, preserving source-literal contents.""" + lines = text.splitlines() + indices, _ = source_boundaries.metadata_indices(lines, marker, adapter) + return [line for index, line in enumerate(lines) if index not in indices] def opening_index(lines: list[str], adapter: LanguageRatioAdapter | None) -> int: @@ -148,11 +150,7 @@ def opening_index(lines: list[str], adapter: LanguageRatioAdapter | None) -> int adapter, the default rule applies: a shebang stays first and the seal follows it. """ - if adapter is not None: - protected = adapter.opening_boundary(lines) - else: - protected = [0] if lines and SHEBANG_RE.match(lines[0].rstrip()) else [] - return max(protected, default=-1) + 1 + return source_boundaries.opening_index(lines, adapter) def place_ratios( @@ -166,8 +164,7 @@ def place_ratios( Returns ``(new_text, changed)``. Existing ratios lines are removed and re-placed. The closing line is the last non-blank line. """ - line_re = _ratios_line_re(marker) - lines = [raw for raw in text.splitlines() if not line_re.match(raw.rstrip())] + lines = strip_ratios_lines(text, marker, adapter) opening = render_ratios_line(marker, values) lines.insert(opening_index(lines, adapter), opening) diff --git a/src/pubskill_lib/source_boundaries.py b/src/pubskill_lib/source_boundaries.py new file mode 100644 index 0000000..be76b25 --- /dev/null +++ b/src/pubskill_lib/source_boundaries.py @@ -0,0 +1,40 @@ +"""Identify examiner metadata only at its reserved source placement boundaries. + +Fence-shaped text elsewhere remains source data, including inside multiline +strings. Entry parsing remains owned by the packaged canonical msdmd parser. +""" +from __future__ import annotations + +import re + + +def opening_index(lines, adapter=None): + if adapter is not None: + protected = adapter.opening_boundary(lines) + else: + protected = [0] if lines and lines[0].startswith("#!") else [] + return max(protected, default=-1) + 1 + + +def metadata_indices(lines, marker, adapter=None): + """Return reserved RATIOS indices and one complete opening NARRATIVE span.""" + ratio_line = re.compile(rf"^{re.escape(marker)}\s*ratios:\s*.+?\s*$") + ratios = set() + opening = opening_index(lines, adapter) + if opening < len(lines) and ratio_line.fullmatch(lines[opening]): + ratios.add(opening) + opening += 1 + closing = len(lines) - 1 + while closing >= 0 and not lines[closing].strip(): + closing -= 1 + if closing >= 0 and ratio_line.fullmatch(lines[closing]): + ratios.add(closing) + narrative = set() + if opening < len(lines) and lines[opening].rstrip() == f"{marker} === NARRATIVE ===": + for index in range(opening + 1, len(lines)): + if not lines[index].startswith(marker): + break + if lines[index].rstrip() == f"{marker} === END NARRATIVE ===": + narrative.update(range(opening, index + 1)) + break + return ratios, narrative diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 04f7464..9332c42 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -47,6 +47,67 @@ def test_blank_model_override_uses_provider_default(self): class NarrativeBoundaryTests(unittest.TestCase): + def test_fence_shaped_literal_data_remains_source(self): + class FakeProvider: + name, model = "fake", "model-1" + def chat(self, system, user): + return "Stores a literal string." + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "literal.py" + original = 'payload = """\n# === NARRATIVE ===\n# id: literal_data\n# summary: alpha\n# === END NARRATIVE ===\n# ratios: loc_comments=1:2 imports_exports=3:4 calls_definitions=5:6\n"""\n' + path.write_text(original) + before = evidence.read_evidence(root, path) + self.assertEqual([], before.narrative_entries) + path.write_text(original.replace("alpha", "beta")) + self.assertNotEqual(before.sha256, evidence.read_evidence(root, path).sha256) + path.write_text(original) + examine._apply(root, [before], [FakeProvider()], True) + namespace = {} + exec(compile(path.read_bytes(), str(path), "exec"), namespace) + expected = {} + exec(compile(original, str(path), "exec"), expected) + self.assertEqual(expected["payload"], namespace["payload"]) + after = evidence.read_evidence(root, path) + self.assertEqual(before.sha256, after.sha256) + self.assertEqual(1, len(after.narrative_entries)) + first = path.read_bytes() + examine._apply(root, [after], [], False) + self.assertEqual(first, path.read_bytes()) + + def test_provider_cannot_overwrite_a_concurrent_source_edit(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "concurrent.py" + path.write_text("print('old')\n") + concurrent = b"print('concurrent edit')\n" + class EditingProvider: + name, model = "fake", "model-1" + def chat(self, system, user): + path.write_bytes(concurrent) + return "Prints old." + _, report = examine._apply(root, [evidence.read_evidence(root, path)], [EditingProvider()], True) + self.assertEqual(concurrent, path.read_bytes()) + self.assertEqual([], report["changed"]) + self.assertIn("concurrent.py", report["hmmm"]) + + def test_failed_publication_preserves_source_and_external_hardlinks(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + original = b"original\n" + path.write_bytes(original) + alias = Path(tmp) / "external.py" + alias.hardlink_to(path) + with patch("pubskill_lib.msdmd_writer.os.replace", side_effect=OSError("publication failed")): + with self.assertRaises(OSError): + msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + self.assertEqual(original, path.read_bytes()) + self.assertEqual([], list(Path(tmp).glob(".examiner-*"))) + msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + self.assertEqual(original, alias.read_bytes()) + self.assertEqual(b"new\n", path.read_bytes()) + def test_apply_preserves_packaged_canonical_parser(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -336,6 +397,23 @@ def test_inspector_endpoint_requires_equals_in_node_24(self): # Official Node v24.15.0 attempts to load 9229 as the entry file here. self.assertEqual(["9229"], list(audit._local_script_targets("node --inspect 9229 app.js"))) + def test_node_inspect_subcommand_and_malformed_urls(self): + self.assertEqual(["missing.js"], list(audit._local_script_targets("node inspect missing.js"))) + self.assertEqual(["missing.js"], list(audit._local_script_targets("node inspect --port=9000 missing.js"))) + self.assertEqual([], list(audit._local_script_targets("node inspect localhost:9229"))) + self.assertEqual([], list(audit._local_script_targets("node inspect -p 1234"))) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "inspect").mkdir() + (root / "package.json").write_text(json.dumps({"scripts": { + "inspect": "node inspect missing.js", + "nul": "node --entry-url file:///tmp/%00.js", + "raw-nul": "node bad\u0000name.js", + }})) + document = audit.audit_path(root, "pin") + self.assertTrue(any("missing local file missing.js" in f["claim"] for f in document["findings"])) + self.assertEqual(2, sum("NUL-containing" in item for item in document["hmmm"])) + def test_quoted_segments_and_entrypoint_urls(self): self.assertEqual(["first.js", "second.js"], list(audit._local_script_targets("node --test-name-pattern 'a|b' first.js && node second.js"))) self.assertEqual([], list(audit._local_script_targets("node --entry-url 'data:text/javascript,console.log(1);'"))) From 53e509f73c09a45d6f90a1ecdf16a52ab1de4aa2 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 03:20:17 +0000 Subject: [PATCH 32/39] Preserve source inodes and publish without replacing competing edits --- .gitignore | 2 ++ README.md | 11 ++++++ src/pubskill_lib/audit.py | 32 ++++++++--------- src/pubskill_lib/boundary.py | 2 +- src/pubskill_lib/examine.py | 17 +++++---- src/pubskill_lib/msdmd_writer.py | 62 +++++++++++++++++++++++--------- tests/test_repairs.py | 58 +++++++++++++++++++++++++++--- 7 files changed, 139 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 45125d0..78d3764 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ __pycache__/ .env .env.* !.env.example + +.examiner-originals-*/ diff --git a/README.md b/README.md index 84e78d2..d71f948 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,14 @@ MPL-2.0, same as skill-lib. Changes to MPL-covered files must be published. ## Canon Do not add skills here first. Add them in skill-lib, mark them appropriately, pin the SHA in `SOURCE.md`, then propagate the public slice. + +Source updates preserve the original inode in a private `.examiner-originals-*` +directory beside the file, recorded under `preserved_sources` in the apply report. +These recovery directories are excluded from examiner inventory and should not be +committed. Publication briefly withdraws the old name, then creates the updated +name only if it remains absent; it never replaces a competing live file. A +collision or observed write to the retained original records `hmmm`. Already-open +writers can still change the retained original after the operation; stop editors +and generators before applying, then inspect recovery files before removing them. +This protocol preserves bytes; it does not claim a transactional edit shared with +uncooperative writers or uninterrupted availability to concurrent readers. diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index c529ae4..65ff6a3 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -198,7 +198,7 @@ def _check_pyproject_scripts(target, sink): ) -def _shell_segments(command): +def _shell_segments(command, separators=";&|\n"): """Split direct shell commands while retaining quoted/escaped separators.""" start, quote, escaped = 0, None, False for index, character in enumerate(command): @@ -211,7 +211,7 @@ def _shell_segments(command): quote = None elif character in {"'", '"'}: quote = character - elif character in ";&|\n": + elif character in separators: yield command[start:index] start = index + 1 yield command[start:] @@ -226,6 +226,8 @@ def _entrypoint_target(token, entry_url, unresolved=None): raise ValueError("unsupported file URL authority") if parsed.scheme not in {"", "file"}: return None + if re.search(r"%(?![0-9a-fA-F]{2})|%(?:2[fF]|5[cC])", parsed.path): + raise ValueError("invalid or unsupported encoded URL path separator") target = unquote(parsed.path, errors="strict") if not target or "\0" in target: raise ValueError("empty or NUL-containing path") @@ -250,27 +252,25 @@ def _local_script_targets(command, unresolved=None): tokens = shlex.split(segment) except ValueError: continue + raw_words = [word for word in _shell_segments(segment, " \t\r") if word] + while tokens and raw_words and re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", raw_words[0]): + tokens.pop(0) + raw_words.pop(0) if not tokens or tokens[0] not in LOCAL_SCRIPT_INTERPRETERS: continue interpreter = tokens[0] non_file_modes = NON_FILE_MODES[interpreter] entry_url = False + inspecting = False index = 1 while index < len(tokens): token = tokens[index] - if interpreter == "node" and token == "inspect": - arguments = tokens[index + 1:] - if not arguments: - break - target = arguments[0] - if re.fullmatch(r"[^:]+:\d+", target) or (len(arguments) == 2 and target == "-p" and arguments[1].isdigit()): - break # Attach to an existing debugger/process, not a file. - if re.fullmatch(r"--port=\d+", target): - target = arguments[1] if len(arguments) > 1 else "" - target = _entrypoint_target(target, False, unresolved) - if target is not None: - yield target - break + if interpreter == "node" and token == "inspect" and not inspecting: + inspecting = True + index += 1 + continue + if inspecting and re.fullmatch(r"[^:]+:\d+", token): + break # Remote debugger attachment. if interpreter == "node" and token in {"--entry-url", "--experimental-entry-url"}: entry_url = True index += 1 @@ -333,8 +333,6 @@ def _check_package_scripts(target, sink, unresolved): script_unresolved = [] for raw_path in _local_script_targets(command, script_unresolved): raw_path = raw_path.strip('"\'') - if "://" in raw_path: - continue try: candidate = Path(raw_path) local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() diff --git a/src/pubskill_lib/boundary.py b/src/pubskill_lib/boundary.py index 3335390..74be335 100644 --- a/src/pubskill_lib/boundary.py +++ b/src/pubskill_lib/boundary.py @@ -62,7 +62,7 @@ def iter_files(root: Path, skip: set[str] | None = None) -> list[Path]: skip = set(skip if skip is not None else DEFAULT_SKIP) found: list[Path] = [] for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = sorted(d for d in dirnames if d not in skip) + dirnames[:] = sorted(d for d in dirnames if d not in skip and not d.startswith(".examiner-originals-")) for name in sorted(filenames): path = Path(dirpath) / name try: diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index 1c943d1..b06cc1b 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -31,14 +31,16 @@ def _canonical_artifact(path: Path) -> bool: def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: - supported = [ev for ev in evidence_list if ev.marker is not None and not _canonical_artifact(Path(ev.path))] + engine = ratios.RatiosEngine() + supported = [ev for ev in evidence_list if ev.marker is not None and ev.encoding is not None and engine.adapter_for(Path(ev.path)) is not None and not _canonical_artifact(Path(ev.path))] return { "root": str(root), "files": len(evidence_list), "supported_files": len(supported), "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(Path(ev.path))], "unsupported": [ - {"path": ev.path, "hmmm": ev.hmmm} for ev in evidence_list if ev.marker is None + {"path": ev.path, "hmmm": ev.hmmm or ["no safe metrics/write adapter"]} + for ev in evidence_list if ev not in supported and not _canonical_artifact(Path(ev.path)) ], "ratios_missing": [ev.path for ev in supported if not ev.ratios_lines], "narrative_present": [ev.path for ev in supported if ev.narrative_entries], @@ -55,6 +57,7 @@ def _apply( changed: list[str] = [] unresolved: dict[str, str] = {} preserved_authority: list[str] = [] + preserved_sources: dict[str, str] = {} now = datetime.now(timezone.utc).isoformat() engine = ratios.RatiosEngine() @@ -67,6 +70,7 @@ def _apply( continue adapter = engine.adapter_for(path) if ev.marker is None or adapter is None or ev.encoding is None: + unresolved[ev.path] = "; ".join(ev.hmmm) or "no safe metrics/write adapter; mutation skipped" continue try: raw = path.read_bytes() @@ -108,9 +112,10 @@ def _apply( if path.is_symlink() or boundary.assert_inside(root, path) != path or path.read_bytes() != raw: unresolved[ev.path] = "source changed during examination; mutation skipped" continue - msdmd_writer.write_text_safely(path, new_text, ev.encoding, expected_raw=raw) - except msdmd_writer.SourceChangedError: - unresolved[ev.path] = "source changed during examination; mutation skipped" + original = msdmd_writer.write_text_safely(path, new_text, ev.encoding, expected_raw=raw) + preserved_sources[ev.path] = original.relative_to(root).as_posix() + except msdmd_writer.SourceChangedError as exc: + unresolved[ev.path] = str(exc) continue except OSError as exc: unresolved[ev.path] = f"source unavailable before write; mutation skipped: {exc}" @@ -122,7 +127,7 @@ def _apply( narratives[ev.path] = entry changed.extend(file_changes) - return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved, "preserved_authority": preserved_authority} + return narratives, {"changed": changed, "narrated": len(narratives), "hmmm": unresolved, "preserved_authority": preserved_authority, "preserved_sources": preserved_sources} def main(argv: list[str] | None = None) -> int: diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 5192e33..092f34c 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -65,24 +65,52 @@ class SourceChangedError(RuntimeError): """The live source no longer matches the inventoried bytes.""" -def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, expected_raw: bytes | None = None) -> None: - """Preserve the source encoding and mode; encode before opening for writing.""" +def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, expected_raw: bytes | None = None) -> Path: + """Publish without replacing a live name; retain the original inode. + + There is a short absent-name interval. Publication uses link's atomic + no-replace guarantee. Open writers keep their original inode in recovery + storage, which is deliberately never deleted by this operation. + """ encoded = new_text.encode(encoding) - mode = None - try: - mode = path.stat().st_mode & 0o777 - except OSError: - pass - temporary = None + if path.is_symlink(): + raise SourceChangedError("source became a symlink; mutation skipped") + raw = path.read_bytes() if expected_raw is None else expected_raw + mode = path.stat().st_mode & 0o777 + # A fresh private directory prevents a preexisting recovery path from + # redirecting writes. The caller reports its path; inventory skips it. + recovery = Path(tempfile.mkdtemp(prefix=".examiner-originals-", dir=path.parent)) + original = recovery / "original" + candidate = recovery / "candidate" + candidate.write_bytes(encoded) + candidate.chmod(mode) + moved = False try: - with tempfile.NamedTemporaryFile(mode="wb", dir=path.parent, prefix=".examiner-", delete=False) as stream: - temporary = Path(stream.name) - stream.write(encoded) - if mode is not None: - temporary.chmod(mode) - if expected_raw is not None and (path.is_symlink() or path.read_bytes() != expected_raw): + if path.is_symlink() or path.read_bytes() != raw: raise SourceChangedError("source changed before metadata publication") - os.replace(temporary, path) + os.rename(path, original) + moved = True + if original.is_symlink() or original.read_bytes() != raw: + raise SourceChangedError(f"source changed during publication; preserved at {original}") + try: + os.link(candidate, path) # Atomic create-if-absent; never replace a competing edit. + except FileExistsError as error: + raise SourceChangedError(f"competing source preserved; prior inode at {original}") from error + if original.read_bytes() != raw: + raise SourceChangedError(f"open writer changed original inode; inspect preserved source at {original}") + return original + except BaseException as failure: + if moved: + try: + os.link(original, path, follow_symlinks=False) + except FileExistsError: + pass # Preserve the live name and the recovery inode independently. + except OSError as error: + raise SourceChangedError(f"source retained at {original}; restore failed: {error}") from error + if isinstance(failure, OSError): + raise OSError(f"publication failed; original retained at {original}: {failure}") from failure + raise finally: - if temporary is not None: - temporary.unlink(missing_ok=True) + candidate.unlink(missing_ok=True) + if not moved: + recovery.rmdir() diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 9332c42..a2e7770 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -92,20 +92,47 @@ def chat(self, system, user): self.assertEqual([], report["changed"]) self.assertIn("concurrent.py", report["hmmm"]) + def test_conditional_publication_preserves_competing_writes(self): + import os + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + original = b"original\n" + concurrent = b"concurrent\n" + path.write_bytes(original) + link = os.link + def competing_write(source, target, **kwargs): + if Path(source).name == "candidate": + path.write_bytes(concurrent) + return link(source, target, **kwargs) + with patch("pubskill_lib.msdmd_writer.os.link", side_effect=competing_write): + with self.assertRaises(msdmd_writer.SourceChangedError): + msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + self.assertEqual(concurrent, path.read_bytes()) + self.assertEqual([original], [p.read_bytes() for p in Path(tmp).glob(".examiner-originals-*/original")]) + def test_failed_publication_preserves_source_and_external_hardlinks(self): + import os with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "source.py" original = b"original\n" path.write_bytes(original) alias = Path(tmp) / "external.py" alias.hardlink_to(path) - with patch("pubskill_lib.msdmd_writer.os.replace", side_effect=OSError("publication failed")): + link = os.link + def fail_candidate(source, target, **kwargs): + if Path(source).name == "candidate": + raise OSError("publication failed") + return link(source, target, **kwargs) + with patch("pubskill_lib.msdmd_writer.os.link", side_effect=fail_candidate): with self.assertRaises(OSError): msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) self.assertEqual(original, path.read_bytes()) - self.assertEqual([], list(Path(tmp).glob(".examiner-*"))) - msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) - self.assertEqual(original, alias.read_bytes()) + with path.open("r+b") as writer: + recovery = msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + writer.write(b"late edit") + writer.truncate() + self.assertEqual(b"late edit", recovery.read_bytes()) + self.assertEqual(b"late edit", alias.read_bytes()) self.assertEqual(b"new\n", path.read_bytes()) def test_apply_preserves_packaged_canonical_parser(self): @@ -192,6 +219,8 @@ def test_apply_skips_parser_supported_language_without_safe_ratio_adapter(self): _, report = examine._apply(root, [ev], [], False) self.assertEqual(original, path.read_text(encoding="utf-8")) self.assertEqual([], report["changed"]) + self.assertIn("index.php", report["hmmm"]) + self.assertEqual(0, examine._plan(root, [ev])["supported_files"]) def test_apply_preserves_encoding_and_source_identity(self): @@ -397,6 +426,27 @@ def test_inspector_endpoint_requires_equals_in_node_24(self): # Official Node v24.15.0 attempts to load 9229 as the entry file here. self.assertEqual(["9229"], list(audit._local_script_targets("node --inspect 9229 app.js"))) + def test_assignment_prefixes_inspect_options_and_url_paths(self): + commands = ( + "NODE_ENV=production node missing.js", + "A='value with spaces' B=two node missing.js", + "node inspect --trace-warnings missing.js", + "node inspect --require preload.js missing.js", + "node inspect --port=9000 --require preload.js missing.js", + ) + for command in commands: + self.assertEqual(["missing.js"], list(audit._local_script_targets(command)), command) + self.assertEqual([], list(audit._local_script_targets("'A=literal-command' node missing.js"))) + for operand in ("file:missing%2Fpart.js", "file:missing%5Cpart.js", "file:missing%ZZ.js"): + unresolved = [] + self.assertEqual([], list(audit._local_script_targets("node --entry-url " + operand, unresolved))) + self.assertTrue(unresolved) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "package.json").write_text(json.dumps({"scripts": {"url": "node https://example.invalid/missing.js"}})) + claims = [item["claim"] for item in audit.audit_path(root, "pin")["findings"] if item["surface"] == "deps"] + self.assertTrue(any("missing local file https://example.invalid/missing.js" in claim for claim in claims), claims) + def test_node_inspect_subcommand_and_malformed_urls(self): self.assertEqual(["missing.js"], list(audit._local_script_targets("node inspect missing.js"))) self.assertEqual(["missing.js"], list(audit._local_script_targets("node inspect --port=9000 missing.js"))) From 363e398b7759412f143b322ff6bb2a052424af21 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 03:36:14 +0000 Subject: [PATCH 33/39] Enforce canonical placement and expose safe source recovery --- README.md | 11 +++++--- src/pubskill_lib/examine.py | 39 +++++++++++++++++++++----- src/pubskill_lib/msdmd_writer.py | 7 +++++ src/pubskill_lib/ratios.py | 7 +++++ tests/test_examine.py | 26 +++++++++++------- tests/test_idempotence.py | 2 +- tests/test_repairs.py | 47 ++++++++++++++++++-------------- 7 files changed, 96 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index d71f948..4c3df32 100644 --- a/README.md +++ b/README.md @@ -76,9 +76,11 @@ OPENAI_BASE_URL / ANTHROPIC_BASE_URL # process environment only Multiple configured providers are attempted sequentially as fallback. Unsupported or not-faithfully-computable metrics remain `hmmm`; they are not guessed. -Python coding cookies and UTF-8 byte-order marks are preserved during source -mutation. If generated prose cannot be encoded in the source encoding, the file -is left intact and the apply report records `hmmm`. Existing narratives remain +UTF-8 byte-order marks are preserved during source mutation. Files whose protected +coding cookies conflict with the pinned canonical RATIOS placement are left intact +and reported as `hmmm`; this consumer cannot expand canonical placement rules. +If generated prose cannot be encoded in the source encoding, the file is also +left intact with `hmmm`. Existing narratives remain available in assembled documentation even when a file has no safe mutation adapter. ## License @@ -92,7 +94,8 @@ Do not add skills here first. Add them in skill-lib, mark them appropriately, pi Source updates preserve the original inode in a private `.examiner-originals-*` directory beside the file, recorded under `preserved_sources` in the apply report. These recovery directories are excluded from examiner inventory and should not be -committed. Publication briefly withdraws the old name, then creates the updated +committed. The required hard-link operations are probed before source is moved. +Publication briefly withdraws the old name, then creates the updated name only if it remains absent; it never replaces a competing live file. A collision or observed write to the retained original records `hmmm`. Already-open writers can still change the retained original after the operation; stop editors diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index b06cc1b..e177615 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -32,16 +32,30 @@ def _canonical_artifact(path: Path) -> bool: def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: engine = ratios.RatiosEngine() - supported = [ev for ev in evidence_list if ev.marker is not None and ev.encoding is not None and engine.adapter_for(Path(ev.path)) is not None and not _canonical_artifact(Path(ev.path))] + supported = [] + unsupported = [] + for ev in evidence_list: + if _canonical_artifact(Path(ev.path)): + continue + reason = "; ".join(ev.hmmm) + if ev.marker is None or ev.encoding is None or engine.adapter_for(Path(ev.path)) is None: + reason = reason or "no safe metrics/write adapter" + else: + try: + path = boundary.assert_inside(root, root / ev.path) + engine.place(path.read_bytes().decode(ev.encoding), ev.marker, {}, path) + except (OSError, UnicodeError, ratios.UnsupportedPlacementError) as error: + reason = str(error) + else: + supported.append(ev) + continue + unsupported.append({"path": ev.path, "hmmm": [reason]}) return { "root": str(root), "files": len(evidence_list), "supported_files": len(supported), "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(Path(ev.path))], - "unsupported": [ - {"path": ev.path, "hmmm": ev.hmmm or ["no safe metrics/write adapter"]} - for ev in evidence_list if ev not in supported and not _canonical_artifact(Path(ev.path)) - ], + "unsupported": unsupported, "ratios_missing": [ev.path for ev in supported if not ev.ratios_lines], "narrative_present": [ev.path for ev in supported if ev.narrative_entries], } @@ -82,6 +96,11 @@ def _apply( unresolved[ev.path] = f"source unavailable; mutation skipped: {exc}" continue + try: + engine.place(original_text, ev.marker, {}, path) + except ratios.UnsupportedPlacementError as error: + unresolved[ev.path] = str(error) + continue new_text = original_text file_changes: list[str] = [] entry = narratives.get(ev.path) @@ -103,7 +122,11 @@ def _apply( file_changes.append(f"{ev.path}:narrative") values = engine.compute(path, evidence.source_text(new_text, ev.marker, path)) - new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) + try: + new_text, ratio_changed = engine.place(new_text, ev.marker, values, path) + except ratios.UnsupportedPlacementError as error: + unresolved[ev.path] = str(error) + continue if ratio_changed: file_changes.append(f"{ev.path}:ratios") @@ -171,11 +194,13 @@ def main(argv: list[str] | None = None) -> int: volume = assemble.assemble_docs(root, evidence_list, narratives, out_dir) if args.json: - print(json.dumps({"changed": report["changed"], "hmmm": report["hmmm"], "volume": str(volume)}, indent=2)) + print(json.dumps({**report, "volume": str(volume)}, indent=2)) else: print(f"applied: {len(report['changed'])} writes") for change in report["changed"]: print(f" {change}") + for path, original in report["preserved_sources"].items(): + print(f" preserved source: {path}: {original}") for path, reason in report["hmmm"].items(): print(f" hmmm: {path}: {reason}") print(f"assembled: {volume}") diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 092f34c..5318a5a 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -86,6 +86,12 @@ def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, exp candidate.chmod(mode) moved = False try: + # Both candidate publication and original restoration require links. + # Probe the same files/directory before withdrawing the live name. + for source in (candidate, path): + probe = recovery / "link-probe" + os.link(source, probe, follow_symlinks=False) + probe.unlink() if path.is_symlink() or path.read_bytes() != raw: raise SourceChangedError("source changed before metadata publication") os.rename(path, original) @@ -112,5 +118,6 @@ def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, exp raise finally: candidate.unlink(missing_ok=True) + (recovery / "link-probe").unlink(missing_ok=True) if not moved: recovery.rmdir() diff --git a/src/pubskill_lib/ratios.py b/src/pubskill_lib/ratios.py index 3d523c5..1e08484 100644 --- a/src/pubskill_lib/ratios.py +++ b/src/pubskill_lib/ratios.py @@ -153,6 +153,10 @@ def opening_index(lines: list[str], adapter: LanguageRatioAdapter | None) -> int return source_boundaries.opening_index(lines, adapter) +class UnsupportedPlacementError(ValueError): + """Protected source lines conflict with the pinned canonical seal boundary.""" + + def place_ratios( text: str, marker: str, @@ -176,6 +180,9 @@ def place_ratios( new_text = "\n".join(lines) if lines: new_text += "\n" + from . import _msdmd_universal + if _msdmd_universal.ratios_placement(new_text, marker) != (True, True): + raise UnsupportedPlacementError("protected source prologue conflicts with pinned canonical RATIOS placement; mutation skipped") return new_text, new_text != text diff --git a/tests/test_examine.py b/tests/test_examine.py index 541e1f5..6500ac1 100644 --- a/tests/test_examine.py +++ b/tests/test_examine.py @@ -1,3 +1,4 @@ +import json import os import shutil import subprocess @@ -87,16 +88,11 @@ def test_python_opening_boundary_respects_encoding_header(self): self.assertEqual([0, 1], adapter.opening_boundary(lines)) self.assertEqual(2, ratios.opening_index(lines, adapter)) - new, _ = ratios.place_ratios( - "\n".join(lines) + "\n", - "#", - {"loc_comments": "1:0", "imports_exports": "1:0", "calls_definitions": "0:0"}, - adapter, - ) - out = new.splitlines() - self.assertTrue(out[0].startswith("#!")) - self.assertIn("coding", out[1]) - self.assertTrue(out[2].startswith("# ratios:")) + with self.assertRaises(ratios.UnsupportedPlacementError): + ratios.place_ratios( + "\n".join(lines) + "\n", "#", + {"loc_comments": "1:0", "imports_exports": "1:0", "calls_definitions": "0:0"}, adapter, + ) def test_find_internal_dependencies_python(self): from pubskill_lib import ratios_adapters @@ -169,10 +165,20 @@ def test_dry_run_reports_without_writing(self): self.assertEqual(0, result.returncode, result.stderr) self.assertEqual(before, (self.root / "tool.py").read_text()) + def test_json_apply_reports_recovery_paths(self): + result = self._run("--apply", "--json") + self.assertEqual(0, result.returncode, result.stderr) + report = json.loads(result.stdout) + self.assertTrue(report["preserved_sources"]) + for relative in report["preserved_sources"].values(): + self.assertTrue((self.root / relative).is_file()) + def test_apply_writes_ratios_and_assembles_docs(self): shell_before = (self.root / "run.sh").read_bytes() result = self._run("--apply", "--out", "docs/examiner") self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("preserved source:", result.stdout) + self.assertIn(".examiner-originals-", result.stdout) tool = (self.root / "tool.py").read_text().splitlines() self.assertTrue(tool[0].startswith("#!")) diff --git a/tests/test_idempotence.py b/tests/test_idempotence.py index 170a1f8..1aaed4d 100644 --- a/tests/test_idempotence.py +++ b/tests/test_idempotence.py @@ -19,7 +19,6 @@ def chat(self, system, user): source = root / "tool.py" source.write_text( "#!/usr/bin/env python3\n" - "# -*- coding: utf-8 -*-\n" "print('hi')\n", encoding="utf-8", ) @@ -28,6 +27,7 @@ def chat(self, system, user): _, first_report = examine._apply(root, first_evidence, [FakeProvider()], True) first_output = source.read_text(encoding="utf-8") self.assertTrue(first_report["changed"]) + self.assertEqual((True, True), evidence._canonical_msdmd.ratios_placement(first_output)) second_evidence = evidence.inventory(root) self.assertEqual(1, len(second_evidence)) diff --git a/tests/test_repairs.py b/tests/test_repairs.py index a2e7770..9e79d92 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -101,7 +101,7 @@ def test_conditional_publication_preserves_competing_writes(self): path.write_bytes(original) link = os.link def competing_write(source, target, **kwargs): - if Path(source).name == "candidate": + if Path(source).name == "candidate" and Path(target) == path: path.write_bytes(concurrent) return link(source, target, **kwargs) with patch("pubskill_lib.msdmd_writer.os.link", side_effect=competing_write): @@ -120,7 +120,7 @@ def test_failed_publication_preserves_source_and_external_hardlinks(self): alias.hardlink_to(path) link = os.link def fail_candidate(source, target, **kwargs): - if Path(source).name == "candidate": + if Path(source).name == "candidate" and Path(target) == path: raise OSError("publication failed") return link(source, target, **kwargs) with patch("pubskill_lib.msdmd_writer.os.link", side_effect=fail_candidate): @@ -223,30 +223,35 @@ def test_apply_skips_parser_supported_language_without_safe_ratio_adapter(self): self.assertEqual(0, examine._plan(root, [ev])["supported_files"]) - def test_apply_preserves_encoding_and_source_identity(self): - class FakeProvider: - name, model = "fake", "model-1" - def chat(self, system, user): - assert "café" in user - return "Prints café." - - for encoding in ("latin-1", "utf-8-sig"): + def test_encoding_prologues_stay_intact_when_canonical_placement_cannot_close(self): + for encoding, prefix in (("latin-1", "# coding: latin-1\n"), ("utf-8-sig", "#!/usr/bin/env python3\n# coding: utf-8\n")): with self.subTest(encoding=encoding), tempfile.TemporaryDirectory() as tmp: root = Path(tmp) path = root / "tool.py" - text = "# coding: " + ("utf-8" if encoding == "utf-8-sig" else encoding) + "\nprint('café')\n" - path.write_bytes(text.encode(encoding)) + raw = (prefix + "print('café')\n").encode(encoding) + path.write_bytes(raw) before = evidence.read_evidence(root, path) - examine._apply(root, [before], [FakeProvider()], True) - after = evidence.read_evidence(root, path) - self.assertEqual(before.sha256, after.sha256) - self.assertFalse(narrative.is_stale(after.narrative_entries[0], after.sha256)) - self.assertIn("café", path.read_bytes().decode(encoding)) - compile(path.read_bytes(), str(path), "exec") - first = path.read_bytes() - _, report = examine._apply(root, [after], [], False) - self.assertEqual(first, path.read_bytes()) + _, report = examine._apply(root, [before], [], False) + self.assertEqual(raw, path.read_bytes()) self.assertEqual([], report["changed"]) + self.assertIn("canonical RATIOS", report["hmmm"]["tool.py"]) + self.assertEqual(0, examine._plan(root, [before])["supported_files"]) + compile(path.read_bytes(), str(path), "exec") + # Encoding fidelity remains independently checked at the writer. + msdmd_writer.write_text_safely(path, prefix + "print('café updated')\n", encoding, expected_raw=raw) + compile(path.read_bytes(), str(path), "exec") + self.assertIn("café updated", path.read_bytes().decode(encoding)) + + def test_missing_hardlink_support_never_withdraws_live_source(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + original = b"original\n" + path.write_bytes(original) + with patch("pubskill_lib.msdmd_writer.os.link", side_effect=OSError("unsupported")): + with self.assertRaises(OSError): + msdmd_writer.write_text_safely(path, "new\n", expected_raw=original) + self.assertEqual(original, path.read_bytes()) + self.assertEqual([], list(Path(tmp).glob(".examiner-originals-*"))) def test_unrepresentable_narrative_does_not_truncate_source(self): class FakeProvider: From 660d2c808c52b746a8a3543bbff563dea99aa8df Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 03:39:13 +0000 Subject: [PATCH 34/39] Keep recovery CLI regressions valid after fixture annotation --- tests/test_examine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_examine.py b/tests/test_examine.py index 6500ac1..669b8b1 100644 --- a/tests/test_examine.py +++ b/tests/test_examine.py @@ -166,6 +166,7 @@ def test_dry_run_reports_without_writing(self): self.assertEqual(before, (self.root / "tool.py").read_text()) def test_json_apply_reports_recovery_paths(self): + (self.root / "recovery_probe.py").write_text("print('fresh source')\n") result = self._run("--apply", "--json") self.assertEqual(0, result.returncode, result.stderr) report = json.loads(result.stdout) @@ -174,6 +175,7 @@ def test_json_apply_reports_recovery_paths(self): self.assertTrue((self.root / relative).is_file()) def test_apply_writes_ratios_and_assembles_docs(self): + (self.root / "recovery_probe.py").write_text("print('fresh source')\n") shell_before = (self.root / "run.sh").read_bytes() result = self._run("--apply", "--out", "docs/examiner") self.assertEqual(0, result.returncode, result.stderr) From 751dbd866a6658cb4fbd95276c473639eceb6e98 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:02:58 +0000 Subject: [PATCH 35/39] Close audit context gaps and preserve source metadata and stale evidence --- README.md | 8 ++++ src/pubskill_lib/audit.py | 64 ++++++++++++++++++++++----- src/pubskill_lib/examine.py | 21 ++++++--- src/pubskill_lib/msdmd_writer.py | 32 +++++++++++--- tests/test_repairs.py | 74 ++++++++++++++++++++++++++++++++ 5 files changed, 179 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4c3df32..82c6884 100644 --- a/README.md +++ b/README.md @@ -102,3 +102,11 @@ writers can still change the retained original after the operation; stop editors and generators before applying, then inspect recovery files before removing them. This protocol preserves bytes; it does not claim a transactional edit shared with uncooperative writers or uninterrupted availability to concurrent readers. + +Source publication currently requires Linux inode metadata support. Ownership, +permission bits, ACL/xattr/security-label bytes are copied and compared before +publication; an unavailable operation leaves the source intact with `hmmm`. +The generated volume uses a new source inventory after application, so preserved +concurrent edits can mark their older narratives stale. +Direct-script audit is intentionally bounded: unsupported commands, malformed +quoting, and working-directory transitions remain visible as `hmmm`. diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 65ff6a3..2e81754 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -30,12 +30,20 @@ PIN_PATTERN = re.compile(r"`([0-9a-f]{40})`") LOCAL_SCRIPT_INTERPRETERS = {"node", "python", "python3", "bash", "sh"} NON_FILE_MODES = { - "node": {"-e", "--eval", "-p", "--print", "--run"}, - "python": {"-c", "-m"}, - "python3": {"-c", "-m"}, - "bash": {"-c"}, - "sh": {"-c"}, + "node": {"-e", "--eval", "-p", "--print", "--run", "-h", "--help", "-v", "--version", "--v8-options", "--completion-bash"}, + "python": {"-c", "-m", "-h", "-?", "--help", "-V", "--version", "--help-env", "--help-xoptions", "--help-all"}, + "python3": {"-c", "-m", "-h", "-?", "--help", "-V", "--version", "--help-env", "--help-xoptions", "--help-all"}, + "bash": {"-c", "--help", "--version"}, + "sh": {"-c", "--help", "--version"}, } +BOOLEAN_OPTIONS = { + "node": {"--trace-warnings", "--inspect", "--inspect-brk", "--inspect-wait", "--watch", "--test", "--no-warnings", "--enable-source-maps", "--experimental-strip-types", "--experimental-transform-types", "--abort-on-uncaught-exception", "--check", "--interactive", "-c", "-i"}, + "python": {"-" + character for character in "bBdEiIOPqRsSuvx"}, + "python3": {"-" + character for character in "bBdEiIOPqRsSuvx"}, + "bash": {"-" + character for character in "abefhkmnptuvxBCEHPTlirs"} | {"+" + character for character in "abefhkmnptuvxBCEHPTlirs"} | {"--debugger", "--dump-po-strings", "--dump-strings", "--noprofile", "--norc", "--posix", "--restricted", "--verbose", "--login"}, + "sh": {"-" + character for character in "aefnuvxCImps"} | {"+" + character for character in "aefnuvxCImps"}, +} + VALUE_OPTIONS = { "python": {"-W", "-X", "--check-hash-based-pycs"}, "python3": {"-W", "-X", "--check-hash-based-pycs"}, @@ -200,8 +208,13 @@ def _check_pyproject_scripts(target, sink): def _shell_segments(command, separators=";&|\n"): """Split direct shell commands while retaining quoted/escaped separators.""" - start, quote, escaped = 0, None, False + start, quote, escaped, comment = 0, None, False, False for index, character in enumerate(command): + if comment: + if character == "\n": + comment = False + start = index + 1 + continue if escaped: escaped = False elif character == "\\" and quote != "'": @@ -211,10 +224,14 @@ def _shell_segments(command, separators=";&|\n"): quote = None elif character in {"'", '"'}: quote = character + elif character == "#" and (index == 0 or command[index - 1] in " \t\r\n;&|()"): + yield command[start:index] + comment = True elif character in separators: yield command[start:index] start = index + 1 - yield command[start:] + if not comment: + yield command[start:] def _entrypoint_target(token, entry_url, unresolved=None): @@ -245,20 +262,36 @@ def _local_script_targets(command, unresolved=None): Python -W/-X, Bash -o/-O and startup files, and common Node value options consume their arguments; attached values and -- delimiters are supported. """ + cwd_unknown = False for segment in _shell_segments(command): if not segment.strip(): continue try: tokens = shlex.split(segment) - except ValueError: + except ValueError as error: + if unresolved is not None: + unresolved.append(f"unparseable package script: {error}") continue raw_words = [word for word in _shell_segments(segment, " \t\r") if word] while tokens and raw_words and re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", raw_words[0]): tokens.pop(0) raw_words.pop(0) - if not tokens or tokens[0] not in LOCAL_SCRIPT_INTERPRETERS: + if not tokens: + continue + interpreter = Path(tokens[0]).name + if interpreter in {"cd", "pushd", "popd"}: + cwd_unknown = True + if unresolved is not None: + unresolved.append("working-directory change is outside direct-script audit scope") + continue + if interpreter not in LOCAL_SCRIPT_INTERPRETERS: + if unresolved is not None: + unresolved.append(f"command is outside direct interpreter audit scope: {tokens[0]!r}") + continue + if cwd_unknown: + if unresolved is not None: + unresolved.append(f"script target after working-directory change is unresolved: {segment.strip()!r}") continue - interpreter = tokens[0] non_file_modes = NON_FILE_MODES[interpreter] entry_url = False inspecting = False @@ -287,6 +320,12 @@ def _local_script_targets(command, unresolved=None): index += 2 continue if token.startswith("-") or (interpreter in {"bash", "sh"} and token.startswith("+")): + if token.startswith("--"): + option = token.split("=", 1)[0] + if option not in BOOLEAN_OPTIONS[interpreter] and option not in VALUE_OPTIONS[interpreter] and not (inspecting and re.fullmatch(r"--port=\d+", token)): + if unresolved is not None: + unresolved.append(f"interpreter option arity is unresolved: {token!r}") + break # Short options may be clustered or carry an attached argument. non_file = False if not token.startswith("--"): @@ -301,6 +340,11 @@ def _local_script_targets(command, unresolved=None): if position == len(token) - 1: index += 1 break + if token[0] + option not in BOOLEAN_OPTIONS[interpreter]: + if unresolved is not None: + unresolved.append(f"interpreter option arity is unresolved: {token!r}") + non_file = True + break if non_file: break index += 1 diff --git a/src/pubskill_lib/examine.py b/src/pubskill_lib/examine.py index e177615..b7c2926 100644 --- a/src/pubskill_lib/examine.py +++ b/src/pubskill_lib/examine.py @@ -26,8 +26,14 @@ from . import ratios -def _canonical_artifact(path: Path) -> bool: - return path.name == "_msdmd_universal.py" and path.parent.name == "pubskill_lib" +def _canonical_artifact(root: Path, path: Path) -> bool: + candidate = path if path.is_absolute() else root / path + try: + return (candidate.relative_to(root).as_posix() == "src/pubskill_lib/_msdmd_universal.py" + and not candidate.is_symlink() + and candidate.read_bytes() == Path(evidence._canonical_msdmd.__file__).read_bytes()) + except (OSError, ValueError): + return False def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: @@ -35,7 +41,7 @@ def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: supported = [] unsupported = [] for ev in evidence_list: - if _canonical_artifact(Path(ev.path)): + if _canonical_artifact(root, Path(ev.path)): continue reason = "; ".join(ev.hmmm) if ev.marker is None or ev.encoding is None or engine.adapter_for(Path(ev.path)) is None: @@ -54,7 +60,7 @@ def _plan(root: Path, evidence_list: list[evidence.FileEvidence]) -> dict: "root": str(root), "files": len(evidence_list), "supported_files": len(supported), - "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(Path(ev.path))], + "preserved_authority": [ev.path for ev in evidence_list if _canonical_artifact(root, Path(ev.path))], "unsupported": unsupported, "ratios_missing": [ev.path for ev in supported if not ev.ratios_lines], "narrative_present": [ev.path for ev in supported if ev.narrative_entries], @@ -79,7 +85,7 @@ def _apply( path = boundary.assert_inside(root, root / ev.path) if ev.narrative_entries: narratives[ev.path] = ev.narrative_entries[0] - if _canonical_artifact(path): + if _canonical_artifact(root, path): preserved_authority.append(ev.path) continue adapter = engine.adapter_for(path) @@ -190,6 +196,11 @@ def main(argv: list[str] | None = None) -> int: return 3 narratives, report = _apply(root, evidence_list, provider_list, args.narrate) + # Rendering observes live source after every write/skip, so an old summary + # cannot retain a current marker after a concurrent edit was preserved. + evidence_list = evidence.inventory(root) + narratives = {ev.path: ev.narrative_entries[0] for ev in evidence_list if ev.narrative_entries} + report["narrated"] = len(narratives) out_dir = boundary.assert_inside(root, root / args.out) volume = assemble.assemble_docs(root, evidence_list, narratives, out_dir) diff --git a/src/pubskill_lib/msdmd_writer.py b/src/pubskill_lib/msdmd_writer.py index 5318a5a..3c4464a 100644 --- a/src/pubskill_lib/msdmd_writer.py +++ b/src/pubskill_lib/msdmd_writer.py @@ -9,6 +9,7 @@ from pathlib import Path import os +import sys import tempfile from . import ratios @@ -65,6 +66,27 @@ class SourceChangedError(RuntimeError): """The live source no longer matches the inventoried bytes.""" +def _inode_metadata(path: Path) -> tuple: + if not sys.platform.startswith("linux") or not hasattr(os, "listxattr"): + raise OSError("source inode metadata verification is unsupported on this platform") + info = path.stat(follow_symlinks=False) + attributes = {name: os.getxattr(path, name, follow_symlinks=False) + for name in os.listxattr(path, follow_symlinks=False)} + return info.st_uid, info.st_gid, info.st_mode & 0o7777, attributes + + +def _copy_inode_metadata(path: Path, metadata: tuple) -> None: + uid, gid, mode, attributes = metadata + os.chown(path, uid, gid, follow_symlinks=False) + path.chmod(mode) + for name in set(os.listxattr(path, follow_symlinks=False)) - attributes.keys(): + os.removexattr(path, name, follow_symlinks=False) + for name, value in attributes.items(): + os.setxattr(path, name, value, follow_symlinks=False) + if _inode_metadata(path) != metadata: + raise OSError("source inode metadata cannot be preserved exactly") + + def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, expected_raw: bytes | None = None) -> Path: """Publish without replacing a live name; retain the original inode. @@ -76,27 +98,27 @@ def write_text_safely(path: Path, new_text: str, encoding: str = "utf-8", *, exp if path.is_symlink(): raise SourceChangedError("source became a symlink; mutation skipped") raw = path.read_bytes() if expected_raw is None else expected_raw - mode = path.stat().st_mode & 0o777 + metadata = _inode_metadata(path) # A fresh private directory prevents a preexisting recovery path from # redirecting writes. The caller reports its path; inventory skips it. recovery = Path(tempfile.mkdtemp(prefix=".examiner-originals-", dir=path.parent)) original = recovery / "original" candidate = recovery / "candidate" - candidate.write_bytes(encoded) - candidate.chmod(mode) moved = False try: + candidate.write_bytes(encoded) + _copy_inode_metadata(candidate, metadata) # Both candidate publication and original restoration require links. # Probe the same files/directory before withdrawing the live name. for source in (candidate, path): probe = recovery / "link-probe" os.link(source, probe, follow_symlinks=False) probe.unlink() - if path.is_symlink() or path.read_bytes() != raw: + if path.is_symlink() or path.read_bytes() != raw or _inode_metadata(path) != metadata: raise SourceChangedError("source changed before metadata publication") os.rename(path, original) moved = True - if original.is_symlink() or original.read_bytes() != raw: + if original.is_symlink() or original.read_bytes() != raw or _inode_metadata(original) != metadata: raise SourceChangedError(f"source changed during publication; preserved at {original}") try: os.link(candidate, path) # Atomic create-if-absent; never replace a competing edit. diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 9e79d92..89cb6c3 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -135,6 +135,68 @@ def fail_candidate(source, target, **kwargs): self.assertEqual(b"late edit", alias.read_bytes()) self.assertEqual(b"new\n", path.read_bytes()) + def test_unrelated_similarly_named_source_is_not_canonical_authority(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name in ("other/pubskill_lib/_msdmd_universal.py", "src/pubskill_lib/_msdmd_universal.py"): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("print('ordinary user code')\n") + item = evidence.read_evidence(root, path) + self.assertEqual(1, examine._plan(root, [item])["supported_files"]) + _, report = examine._apply(root, [item], [], False) + self.assertTrue(report["changed"]) + self.assertEqual([], report["preserved_authority"]) + + def test_candidate_setup_failure_cleans_recovery_storage(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + path.write_text("original\n") + for operation in ("write_bytes", "chmod"): + with patch.object(Path, operation, side_effect=OSError("quota")): + with self.assertRaises(OSError): + msdmd_writer.write_text_safely(path, "new\n") + self.assertEqual("original\n", path.read_text()) + self.assertEqual([], list(Path(tmp).glob(".examiner-originals-*"))) + + def test_publication_preserves_inode_metadata_or_refuses_to_move(self): + import os + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source.py" + path.write_text("original\n") + path.chmod(0o751) + os.setxattr(path, "user.pubskill_test", b"retained") + original_metadata = msdmd_writer._inode_metadata(path) + msdmd_writer.write_text_safely(path, "new\n") + self.assertEqual(original_metadata, msdmd_writer._inode_metadata(path)) + with patch("pubskill_lib.msdmd_writer.os.setxattr", side_effect=OSError("metadata denied")): + with self.assertRaises(OSError): + msdmd_writer.write_text_safely(path, "third\n") + self.assertEqual("new\n", path.read_text()) + self.assertEqual(original_metadata, msdmd_writer._inode_metadata(path)) + + def test_assembled_narrative_is_stale_after_preserving_concurrent_edit(self): + import contextlib, io + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "source.py" + original = "print('old')\n" + path.write_text(original) + ev = evidence.read_evidence(root, path) + decorated, _ = msdmd_writer.upsert_narrative(original, "#", {"id": "old_narrative", "summary": "Old summary", "evidence_sha256": ev.sha256}, path) + path.write_text(decorated) + class EditingProvider: + name, model = "fake", "model" + def chat(self, system, user): + path.write_text(path.read_text().replace("print('old')", "print('edited')")) + return "Generated stale summary" + with patch("pubskill_lib.providers.configured_providers", return_value=[EditingProvider()]), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(0, examine.main(["--repo", str(root), "--apply", "--narrate"])) + output = (root / "docs/examiner/EXAMINER.md").read_text() + self.assertIn("Old summary", output) + self.assertIn("> stale:", output) + self.assertIn("print('edited')", path.read_text()) + def test_apply_preserves_packaged_canonical_parser(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -486,6 +548,18 @@ def test_quoted_segments_and_entrypoint_urls(self): self.assertTrue(any("escapes repository via /definitely/missing.js" in claim for claim in claims)) self.assertTrue(any("missing local file missing.js" in claim for claim in claims)) + def test_exit_comments_paths_and_unresolved_shell_context(self): + for command in ("python --help missing.py", "python -uV missing.py", "node --version missing.js", "bash --help missing.sh"): + self.assertEqual([], list(audit._local_script_targets(command)), command) + self.assertEqual(["real.js"], list(audit._local_script_targets("node real.js # disabled && node missing.js"))) + self.assertEqual(["real.js", "next.js"], list(audit._local_script_targets("node real.js # disabled && node missing.js\nnode next.js"))) + self.assertEqual(["missing.js"], list(audit._local_script_targets("/usr/bin/node missing.js"))) + self.assertEqual(["missing.py"], list(audit._local_script_targets("./venv/bin/python missing.py"))) + for command in ("node 'missing.js", "cd frontend && node build.js", "node --unknown-option value missing.js", "unknown-runner missing.js"): + unresolved = [] + self.assertEqual([], list(audit._local_script_targets(command, unresolved)), command) + self.assertTrue(unresolved, command) + def test_non_object_package_manifest_is_target_defect(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From 5b0f03fc95d48416db803040e7017682cea15969 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:10:18 +0000 Subject: [PATCH 36/39] Keep unresolved shell expansion contexts out of literal path findings --- README.md | 4 +++- src/pubskill_lib/audit.py | 32 ++++++++++++++++++++++++++++++++ tests/test_repairs.py | 3 ++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 82c6884..8cea28e 100644 --- a/README.md +++ b/README.md @@ -109,4 +109,6 @@ publication; an unavailable operation leaves the source intact with `hmmm`. The generated volume uses a new source inventory after application, so preserved concurrent edits can mark their older narratives stale. Direct-script audit is intentionally bounded: unsupported commands, malformed -quoting, and working-directory transitions remain visible as `hmmm`. +quoting, shell expansions/control syntax, and working-directory transitions remain +visible as `hmmm`. Later commands after an unresolved shell context inherit that +uncertainty; the tool does not guess their working directory or entrypoint. diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 2e81754..0d3ffc7 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -234,6 +234,30 @@ def _shell_segments(command, separators=";&|\n"): yield command[start:] +def _shell_context_gap(segment): + """Refuse expansion/control syntax that this literal-path audit cannot resolve.""" + quote, escaped = None, False + for character in segment: + if escaped: + if character == "\n": + return "shell line continuation is outside literal-path audit scope" + escaped = False + continue + if character == "\\" and quote != "'": + escaped = True + elif quote == "'": + if character == "'": + quote = None + elif character in "$`" or (quote is None and character in "*?[]{}()<>~"): + return "shell expansion or control syntax is outside literal-path audit scope" + elif quote: + if character == quote: + quote = None + elif character in {"'", '"'}: + quote = character + return None + + def _entrypoint_target(token, entry_url, unresolved=None): try: target = token @@ -266,9 +290,16 @@ def _local_script_targets(command, unresolved=None): for segment in _shell_segments(command): if not segment.strip(): continue + gap = _shell_context_gap(segment) + if gap: + cwd_unknown = True + if unresolved is not None: + unresolved.append(gap) + continue try: tokens = shlex.split(segment) except ValueError as error: + cwd_unknown = True if unresolved is not None: unresolved.append(f"unparseable package script: {error}") continue @@ -285,6 +316,7 @@ def _local_script_targets(command, unresolved=None): unresolved.append("working-directory change is outside direct-script audit scope") continue if interpreter not in LOCAL_SCRIPT_INTERPRETERS: + cwd_unknown = True if unresolved is not None: unresolved.append(f"command is outside direct interpreter audit scope: {tokens[0]!r}") continue diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 89cb6c3..95c3893 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -555,7 +555,8 @@ def test_exit_comments_paths_and_unresolved_shell_context(self): self.assertEqual(["real.js", "next.js"], list(audit._local_script_targets("node real.js # disabled && node missing.js\nnode next.js"))) self.assertEqual(["missing.js"], list(audit._local_script_targets("/usr/bin/node missing.js"))) self.assertEqual(["missing.py"], list(audit._local_script_targets("./venv/bin/python missing.py"))) - for command in ("node 'missing.js", "cd frontend && node build.js", "node --unknown-option value missing.js", "unknown-runner missing.js"): + self.assertEqual(["$literal.js"], list(audit._local_script_targets("node '$literal.js'"))) + for command in ("node 'missing.js", "cd frontend && node build.js", "node --unknown-option value missing.js", "unknown-runner missing.js", "node $SCRIPT", "node *.js", "(cd frontend && node build.js)", "node < input.js", "node \\\n missing.js"): unresolved = [] self.assertEqual([], list(audit._local_script_targets(command, unresolved)), command) self.assertTrue(unresolved, command) From e065432294f705317e7825d397b09d8f76f0e247 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:24:33 +0000 Subject: [PATCH 37/39] Keep literal entrypoints visible around dynamic child arguments --- src/pubskill_lib/audit.py | 32 ++++++++++++++++++++------------ tests/test_repairs.py | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 0d3ffc7..9535997 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -234,10 +234,10 @@ def _shell_segments(command, separators=";&|\n"): yield command[start:] -def _shell_context_gap(segment): - """Refuse expansion/control syntax that this literal-path audit cannot resolve.""" +def _shell_context_gap(segment, *, context_only=False): + """Identify unsupported syntax, separating word expansion from shell structure.""" quote, escaped = None, False - for character in segment: + for index, character in enumerate(segment): if escaped: if character == "\n": return "shell line continuation is outside literal-path audit scope" @@ -248,8 +248,11 @@ def _shell_context_gap(segment): elif quote == "'": if character == "'": quote = None - elif character in "$`" or (quote is None and character in "*?[]{}()<>~"): - return "shell expansion or control syntax is outside literal-path audit scope" + elif quote is None and (character in "`{}()<>" or segment[index:index + 2] == "$("): + return "shell control syntax is outside literal-path audit scope" + elif not context_only and (character in "$`" or (quote is None and + (character in "*?[]" or (character == "~" and (index == 0 or segment[index - 1].isspace()))))): + return "shell word expansion is outside literal-path audit scope" elif quote: if character == quote: quote = None @@ -291,11 +294,10 @@ def _local_script_targets(command, unresolved=None): if not segment.strip(): continue gap = _shell_context_gap(segment) - if gap: - cwd_unknown = True - if unresolved is not None: - unresolved.append(gap) - continue + if gap and unresolved is not None: + unresolved.append(gap) + prior_cwd_unknown = cwd_unknown + cwd_unknown = cwd_unknown or bool(_shell_context_gap(segment, context_only=True)) try: tokens = shlex.split(segment) except ValueError as error: @@ -309,6 +311,9 @@ def _local_script_targets(command, unresolved=None): raw_words.pop(0) if not tokens: continue + if _shell_context_gap(raw_words[0]): + cwd_unknown = True # A dynamic command could resolve to a shell builtin. + continue interpreter = Path(tokens[0]).name if interpreter in {"cd", "pushd", "popd"}: cwd_unknown = True @@ -320,7 +325,7 @@ def _local_script_targets(command, unresolved=None): if unresolved is not None: unresolved.append(f"command is outside direct interpreter audit scope: {tokens[0]!r}") continue - if cwd_unknown: + if prior_cwd_unknown: if unresolved is not None: unresolved.append(f"script target after working-directory change is unresolved: {segment.strip()!r}") continue @@ -329,6 +334,8 @@ def _local_script_targets(command, unresolved=None): inspecting = False index = 1 while index < len(tokens): + if _shell_context_gap(" ".join(raw_words[:index + 1])): + break token = tokens[index] if interpreter == "node" and token == "inspect" and not inspecting: inspecting = True @@ -341,7 +348,8 @@ def _local_script_targets(command, unresolved=None): index += 1 continue if token == "--": - if index + 1 < len(tokens) and tokens[index + 1] != "-": + if (index + 1 < len(tokens) and tokens[index + 1] != "-" + and not _shell_context_gap(" ".join(raw_words[:index + 2]))): target = _entrypoint_target(tokens[index + 1], entry_url, unresolved) if target is not None: yield target diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 95c3893..f26e7f3 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -548,6 +548,26 @@ def test_quoted_segments_and_entrypoint_urls(self): self.assertTrue(any("escapes repository via /definitely/missing.js" in claim for claim in claims)) self.assertTrue(any("missing local file missing.js" in claim for claim in claims)) + def test_word_expansions_preserve_independent_literal_entrypoints(self): + for command, expected in ( + ('node missing.js "$ARG"', ["missing.js"]), + ('python missing.py "$ARG"', ["missing.py"]), + ('node -- missing.js "$ARG"', ["missing.js"]), + ('node build~backup.js', ["build~backup.js"]), + ('node $SCRIPT && node missing.js', ["missing.js"]), + ('node *.js && node missing.js', ["missing.js"]), + ('node "$SCRIPT" && node missing.js', ["missing.js"]), + ('node missing.js "$(pwd)"', ["missing.js"]), + ('node ~/script.js', []), + ('node -- $SCRIPT', []), + ('node $(cd ..; node hidden.js; pwd) && node uncertain.js', []), + ): + with self.subTest(command=command): + gaps = [] + self.assertEqual(list(audit._local_script_targets(command, gaps)), expected) + if "$" in command or "*" in command or "~/" in command: + self.assertTrue(gaps) + def test_exit_comments_paths_and_unresolved_shell_context(self): for command in ("python --help missing.py", "python -uV missing.py", "node --version missing.js", "bash --help missing.sh"): self.assertEqual([], list(audit._local_script_targets(command)), command) From 62f863ad92e5daa44b1db58c2813035b6c58bc93 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:45:19 +0000 Subject: [PATCH 38/39] Close shell operand and undecodable evidence hash gaps --- src/pubskill_lib/audit.py | 67 ++++++++++++++++++++++++++++++++---- src/pubskill_lib/evidence.py | 10 +++++- tests/test_repairs.py | 53 ++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 8 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 9535997..375b7ab 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -236,13 +236,16 @@ def _shell_segments(command, separators=";&|\n"): def _shell_context_gap(segment, *, context_only=False): """Identify unsupported syntax, separating word expansion from shell structure.""" - quote, escaped = None, False + quote, escaped, word_start = None, False, 0 for index, character in enumerate(segment): if escaped: if character == "\n": return "shell line continuation is outside literal-path audit scope" escaped = False continue + if quote is None and character.isspace(): + word_start = index + 1 + continue if character == "\\" and quote != "'": escaped = True elif quote == "'": @@ -251,7 +254,9 @@ def _shell_context_gap(segment, *, context_only=False): elif quote is None and (character in "`{}()<>" or segment[index:index + 2] == "$("): return "shell control syntax is outside literal-path audit scope" elif not context_only and (character in "$`" or (quote is None and - (character in "*?[]" or (character == "~" and (index == 0 or segment[index - 1].isspace()))))): + (character in "*?[]" or (character == "~" and (index == word_start or + (re.match(r"[A-Za-z_][A-Za-z0-9_]*=", segment[word_start:index]) + and segment[index - 1] in "=:")))))): return "shell word expansion is outside literal-path audit scope" elif quote: if character == quote: @@ -261,6 +266,50 @@ def _shell_context_gap(segment, *, context_only=False): return None +def _fixed_word_arity(raw): + """Bounded proof that a supported option value remains one shell argument.""" + quote, escaped = None, False + for index, character in enumerate(raw): + if escaped: + escaped = False + continue + if character == "\\" and quote != "'": + escaped = True + elif quote == "'": + if character == "'": + quote = None + elif quote == '"': + if character == '"': + quote = None + elif character == "$": + # Ordinary quoted scalar expansions have fixed arity. Positional + # arrays and complex parameter/substitution forms stay unresolved. + tail = raw[index:] + if not re.match(r"\$(?:[A-Za-z_][A-Za-z0-9_]*|[0-9*#?$!]|\{[A-Za-z_][A-Za-z0-9_]*\})", tail): + return False + elif character == "`": + return False + elif character in {"'", '"'}: + quote = character + elif character in "$`*?[]{}()<>" or character.isspace(): + return False + return quote is None and not escaped + + +def _attached_option_value(token, interpreter): + if token.startswith("--"): + return "=" in token and token.split("=", 1)[0] in VALUE_OPTIONS[interpreter] + if not token.startswith(("-", "+")): + return False + for position, character in enumerate(token[1:], start=1): + option = token[0] + character + if option in VALUE_OPTIONS[interpreter]: + return position < len(token) - 1 + if option not in BOOLEAN_OPTIONS[interpreter]: + return False + return False + + def _entrypoint_target(token, entry_url, unresolved=None): try: target = token @@ -334,9 +383,10 @@ def _local_script_targets(command, unresolved=None): inspecting = False index = 1 while index < len(tokens): - if _shell_context_gap(" ".join(raw_words[:index + 1])): - break token = tokens[index] + if _shell_context_gap(raw_words[index]): + if not _attached_option_value(token, interpreter) or not _fixed_word_arity(raw_words[index]): + break if interpreter == "node" and token == "inspect" and not inspecting: inspecting = True index += 1 @@ -349,7 +399,7 @@ def _local_script_targets(command, unresolved=None): continue if token == "--": if (index + 1 < len(tokens) and tokens[index + 1] != "-" - and not _shell_context_gap(" ".join(raw_words[:index + 2]))): + and not _shell_context_gap(raw_words[index + 1])): target = _entrypoint_target(tokens[index + 1], entry_url, unresolved) if target is not None: yield target @@ -357,6 +407,8 @@ def _local_script_targets(command, unresolved=None): if token == "-" or token.split("=", 1)[0] in non_file_modes: break if token in VALUE_OPTIONS[interpreter]: + if index + 1 < len(raw_words) and not _fixed_word_arity(raw_words[index + 1]): + break index += 2 continue if token.startswith("-") or (interpreter in {"bash", "sh"} and token.startswith("+")): @@ -373,11 +425,13 @@ def _local_script_targets(command, unresolved=None): if interpreter in {"bash", "sh"}: modes.add("s") # Read commands from stdin. for position, option in enumerate(token[1:], start=1): - if token[0] == "-" and option in modes: + if (token[0] == "-" and option in modes) or (interpreter == "bash" and option == "s"): non_file = True break if token[0] + option in VALUE_OPTIONS[interpreter]: if position == len(token) - 1: + if index + 1 < len(raw_words) and not _fixed_word_arity(raw_words[index + 1]): + non_file = True index += 1 break if token[0] + option not in BOOLEAN_OPTIONS[interpreter]: @@ -416,7 +470,6 @@ def _check_package_scripts(target, sink, unresolved): continue script_unresolved = [] for raw_path in _local_script_targets(command, script_unresolved): - raw_path = raw_path.strip('"\'') try: candidate = Path(raw_path) local = candidate.resolve() if candidate.is_absolute() else (target / candidate).resolve() diff --git a/src/pubskill_lib/evidence.py b/src/pubskill_lib/evidence.py index d0c51ff..afac379 100644 --- a/src/pubskill_lib/evidence.py +++ b/src/pubskill_lib/evidence.py @@ -122,7 +122,15 @@ def read_evidence(root: Path, path: Path) -> FileEvidence: item.hmmm.append("metadata-excluding source hash unavailable; mutation disabled") return item - stable_encoded = source_text(text, marker, path).encode("utf-8") + try: + stable_encoded = source_text(text, marker, path).encode("utf-8") + except UnicodeError as error: + item.sha256 = item.raw_sha256 + item.marker = None + item.encoding = None + item.hmmm.append(f"source encoding unresolved while hashing: {error}") + item.hmmm.append("metadata-excluding source hash unavailable; mutation disabled") + return item item.sha256 = hashlib.sha256(stable_encoded).hexdigest() first_line = text.splitlines()[0].rstrip() if text.splitlines() else "" diff --git a/tests/test_repairs.py b/tests/test_repairs.py index f26e7f3..0c6c072 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -379,6 +379,24 @@ def test_raw_sha256_hashes_literal_python_bytes_and_honors_cookie(self): self.assertEqual("#", item.marker) self.assertFalse(item.hmmm) + def test_unencodable_stable_source_hash_is_hmmm_without_mutation(self): + from contextlib import redirect_stdout + import io + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / "surrogate.py" + raw = b"# coding: unicode_escape\n# " + bytes((92,)) + b"ud800\n" + path.write_bytes(raw) + item = evidence.read_evidence(root, path) + self.assertEqual(hashlib.sha256(raw).hexdigest(), item.sha256) + self.assertIsNone(item.encoding) + self.assertIsNone(item.marker) + self.assertTrue(any("encoding unresolved while hashing" in text for text in item.hmmm)) + for options in ([], ["--apply"]): + with redirect_stdout(io.StringIO()): + self.assertEqual(0, examine.main(["--repo", str(root), "--json", *options])) + self.assertEqual(raw, path.read_bytes()) + def test_undecodable_non_python_source_is_hmmm_and_not_mutable(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -568,6 +586,41 @@ def test_word_expansions_preserve_independent_literal_entrypoints(self): if "$" in command or "*" in command or "~/" in command: self.assertTrue(gaps) + def test_fixed_arity_option_values_and_bash_stdin_modes(self): + for command, expected in ( + ('node --require "$PRELOAD" missing.js', ["missing.js"]), + ('node --require="$PRELOAD" missing.js', ["missing.js"]), + ('node -r"$PRELOAD" missing.js', ["missing.js"]), + ('python -W "$WARN" missing.py', ["missing.py"]), + ('python -uW"$WARN" missing.py', ["missing.py"]), + ('python -uW "$WARN" missing.py', ["missing.py"]), + ('node --require "$@" uncertain.js', []), + ('node --require $PRELOAD uncertain.js', []), + ('bash +s missing.sh', []), + ('bash +es missing.sh', []), + ): + with self.subTest(command=command): + self.assertEqual(expected, list(audit._local_script_targets(command))) + + def test_assignment_tildes_and_literal_filename_quotes(self): + for command, expected in ( + ('node foo=~/bar', []), + ('node foo=prefix:~/bar', []), + ('node foo-bar=~/bar', ["foo-bar=~/bar"]), + ('node build~backup.js', ["build~backup.js"]), + ): + gaps = [] + self.assertEqual(expected, list(audit._local_script_targets(command, gaps)), command) + self.assertEqual(not expected, bool(gaps), command) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "present.js").write_text("// ordinary name exists\n") + (root / "package.json").write_text(json.dumps({"scripts": {"quoted": "node " + "'" + '\"present.js\"' + "'"}})) + claims = [finding["claim"] for finding in audit.audit_path(root, "pin")["findings"]] + self.assertTrue(any('missing local file "present.js"' in claim for claim in claims), claims) + (root / '\"present.js\"').write_text("// exact quote-named file exists\n") + self.assertFalse(any("missing local file" in finding["claim"] for finding in audit.audit_path(root, "pin")["findings"])) + def test_exit_comments_paths_and_unresolved_shell_context(self): for command in ("python --help missing.py", "python -uV missing.py", "node --version missing.js", "bash --help missing.sh"): self.assertEqual([], list(audit._local_script_targets(command)), command) From 1b52d085a5c6bae3808727408c5795a4ff684770 Mon Sep 17 00:00:00 2001 From: Erin Spencer Date: Sat, 12 Sep 2026 04:57:33 +0000 Subject: [PATCH 39/39] Restrict assignment tilde expansion to its first separator or colons --- src/pubskill_lib/audit.py | 4 ++-- tests/test_repairs.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pubskill_lib/audit.py b/src/pubskill_lib/audit.py index 375b7ab..e0ad99b 100644 --- a/src/pubskill_lib/audit.py +++ b/src/pubskill_lib/audit.py @@ -255,8 +255,8 @@ def _shell_context_gap(segment, *, context_only=False): return "shell control syntax is outside literal-path audit scope" elif not context_only and (character in "$`" or (quote is None and (character in "*?[]" or (character == "~" and (index == word_start or - (re.match(r"[A-Za-z_][A-Za-z0-9_]*=", segment[word_start:index]) - and segment[index - 1] in "=:")))))): + (re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=", segment[word_start:index]) + or (segment[index - 1] == ":" and re.match(r"[A-Za-z_][A-Za-z0-9_]*=", segment[word_start:index])))))))): return "shell word expansion is outside literal-path audit scope" elif quote: if character == quote: diff --git a/tests/test_repairs.py b/tests/test_repairs.py index 0c6c072..c03c999 100644 --- a/tests/test_repairs.py +++ b/tests/test_repairs.py @@ -606,6 +606,7 @@ def test_assignment_tildes_and_literal_filename_quotes(self): for command, expected in ( ('node foo=~/bar', []), ('node foo=prefix:~/bar', []), + ('node entry=value=~/missing.js', ["entry=value=~/missing.js"]), ('node foo-bar=~/bar', ["foo-bar=~/bar"]), ('node build~backup.js', ["build~backup.js"]), ):