From cbd72633bc10720e761ad8a3dff31bb699f40dda Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 04:38:45 +0200 Subject: [PATCH 1/8] fix: harden v0.4.1 integrity contracts --- CHANGELOG.md | 30 +- CITATION.cff | 2 +- INDEPENDENT_IMPLEMENTATION.md | 8 + LICENSE | 9 +- PRODUCTION_READINESS.md | 6 +- PUBLISHING.md | 14 +- README.md | 13 +- REVIEW_REQUEST.md | 9 + SECURITY.md | 8 +- THREAT_MODEL.md | 5 + VERSION | 2 +- prototype/README.md | 5 + prototype/checkpoints.py | 4 +- prototype/conformance.py | 41 +- prototype/controller.py | 32 +- prototype/signing.py | 4 +- publication/active-surfaces.json | 11 +- readiness/production-readiness.json | 2 +- scripts/check_publication.py | 448 ++++++++---------- scripts/check_readiness.py | 276 ++++++++++- scripts/validate_repo.py | 7 + spec/README.md | 9 + spec/receipt.schema.json | 2 +- .../receipt-bad-state-root-invalid.json | 2 +- spec/vectors/receipt-binding-mutations.json | 4 +- spec/vectors/receipt-failed-valid.json | 4 +- .../receipt-missing-state-root-invalid.json | 2 +- spec/vectors/receipt-partial-valid.json | 4 +- .../receipt-pending-as-result-invalid.json | 4 +- ...eceipt-unknown-coverage-value-invalid.json | 4 +- .../receipt-unknown-disposition-invalid.json | 4 +- spec/vectors/receipt-unknown-valid.json | 4 +- spec/vectors/receipt-verified-valid.json | 4 +- tests/test_checkpoints.py | 6 +- tests/test_conformance.py | 13 +- tests/test_controller.py | 26 +- tests/test_publication.py | 33 +- tests/test_readiness.py | 64 ++- tests/test_schema.py | 5 + tests/test_validate_repo.py | 37 +- 40 files changed, 834 insertions(+), 333 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b2dbf0..92f135e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,35 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), Changes to the bounded novelty statement or the source comparison are recorded here even when they narrow or retire part of the claim. That is the intended direction of travel, not an exception. -## [Unreleased](https://github.com/thomaswillner/llm-errata/compare/v0.4.0...HEAD) +## [Unreleased](https://github.com/thomaswillner/llm-errata/compare/v0.4.1...HEAD) + +## [0.4.1](https://github.com/thomaswillner/llm-errata/releases/tag/v0.4.1) - 2026-08-14 + +Integrity and conformance hardening release following an authorized read-only +Claude Opus 5 adversarial review of v0.4.0. The review recommended keeping +v0.4.0 published and issuing this patch release rather than withdrawing it. + +- Separated the immutable normative review target from the metadata-only + packaging/release commit. Publication and adapter-conformance metadata now + share one target; tag-time validation binds the final release commit without + attempting a self-referential Git hash. +- Scoped the offline publication checker to claims it can prove and made live + GitHub verification explicitly inconclusive without a freshness receipt. +- Enforced ledger-to-matrix criterion equality for all six readiness gates and + added gate-specific, commit-bound independent evidence contracts for G3-G5. +- Widened receipt state roots from truncated 128-bit values to full SHA-256 and + required the same width in durable quarantine checkpoints and vectors. +- Disclosed state-root non-binding for every snapshot-less adapter path, + including opaque and lineage-incomplete stores. +- Clarified that all six normative Markdown contracts are Specification + Materials and that conformance bindings execute trusted code only. +- Replaced the silent pre-corpus digest skip with an explicit failure and bound + the adapter corpus into G2 digests while normalizing only its unavoidable + self-target pointer. + +The production verdict remains **NOT_PROD_READY**. G2-G6 remain `BLOCKED`; this +release adds no external review, independent implementation, operated-system, +cryptography, or operational-readiness evidence. ## [0.4.0](https://github.com/thomaswillner/llm-errata/releases/tag/v0.4.0) - 2026-08-14 diff --git a/CITATION.cff b/CITATION.cff index 9aae94b..69af7c3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,7 +5,7 @@ type: software authors: - family-names: "Willner" given-names: "Thomas Rainer" -version: 0.4.0 +version: 0.4.1 date-released: 2026-08-14 license: LicenseRef-LLM-Errata-Personal-Use license-url: "https://github.com/thomaswillner/llm-errata/blob/main/LICENSE" diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index d52af3a..1a72a27 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -68,6 +68,14 @@ normalization or embedding similarity cannot substitute for identity. A pass is candidate internal evidence only and still requires a separate producer to validate both implementations and the validator. +The binding command executes imported Python inside the invoking process. It is +for **trusted code only** from a reviewed, clean checkout. Git identity, +tracked-byte binding, namespace isolation, and timeouts improve evidence +provenance; they are not an operating-system sandbox and do not make untrusted +candidate code safe. Production evaluation of untrusted submissions requires a +separate least-privilege process or container with explicit filesystem, +network, secret, CPU, and memory limits. + ## Independence and evidence An implementation report must name its authors, repository and commit, supported diff --git a/LICENSE b/LICENSE index d1e21e0..b094cd2 100644 --- a/LICENSE +++ b/LICENSE @@ -9,12 +9,15 @@ versions permanently. This licence does not, and cannot, withdraw it. 1. DEFINITIONS "Specification Materials" means the normative requirements incorporated by - reference from README.md, IDEA.md, ROADMAP.md, SECURITY.md, and - THREAT_MODEL.md, plus files under `spec/` other than `spec/vendor/`. + reference from README.md, IDEA.md, ROADMAP.md, SECURITY.md, THREAT_MODEL.md, + HARD_PROBLEMS.md, PRIOR_ART.md, RESEARCH.md, PRODUCTION_READINESS.md, + INDEPENDENT_IMPLEMENTATION.md, and PHASE3_SYSTEMS.md, plus files under + `spec/` other than `spec/vendor/`. "Reference Code" means files under `prototype/`, `scripts/`, and `tests/`, together with other repository software not expressly included in the - Specification Materials. + Specification Materials. Markdown documentation is not Reference Code merely + because it is not listed as a Specification Material. "Implementation" means independently authored software or services that implement a material part of the Specification Materials without copying diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 2df8d78..92c9619 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -4,7 +4,7 @@ | Field | Value | |---|---| -| Version | 0.4.0 | +| Version | 0.4.1 | | Verdict | **NOT_PROD_READY** | | Ledger | `readiness/production-readiness.json` | @@ -23,9 +23,9 @@ requirement to wait indefinitely for unsolicited external reviewers. |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | | G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, adapter-level conformance, validator anti-vacuity controls, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | -| G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | +| G3 | Production signing uses an audited constant-time library through the Signer seam and independent security review covers key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | Inspeximus `v2.7.0` is one tagged externally authored adapter candidate with disclosed v2.6.1 reference-code contamination and a claimed clean-room rewrite. It targets historical commit `a477fe4f5c86730031b6285d9505778fb8eec060`; provenance, current-target behavior, a second candidate, and a third-party validator result remain unverified. | Rebind candidates to the current immutable target; obtain dated evidence from two independently authored adapters, including separate provenance review where needed, and a separately produced third-party validator result. | -| G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | +| G5 | One user-controlled synthetic root completes the declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | | G6 | All ten operational scopes pass from one independent report bound to exact commit and deployment, with declared thresholds and measured comparators. | `BLOCKED` | No independent report binds an exact commit and deployment to passing measured comparators for all ten operational scopes. | One qualifying independent report with declared workload, platform, failure domain, observation window, numeric thresholds, raw artifacts, and passing measurements for every scope. | ## Approval boundaries diff --git a/PUBLISHING.md b/PUBLISHING.md index 3988c75..cf0d76c 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -15,13 +15,20 @@ Do not call the repository a standard, certified protocol, proven deletion syste ## Before making the repository public +LLM Errata uses two non-interchangeable commit identities. The immutable +`review_target` is the normative source commit. A later packaging commit may +change only `publication/active-surfaces.json` and +`spec/adapter-conformance.json` to point back to that source. The release tag +binds the packaging commit. This avoids requiring a Git commit to contain its +own SHA while keeping normative and release identities auditable. + 1. Run `make check` from the repository root. It must exit zero. Exit `2` from the claim guard is *inconclusive*, not a pass. 2. Run `make links`. Every cited URL must resolve or be reported as `blocked`, never `dead`. 3. Review the author name, date, independent-publication disclaimer, and AI-assisted research disclosure. 4. Check every claim in `PRIOR_ART.md` against the cited primary or authoritative source. 5. Request a public archive snapshot for each source listed as unpinned in `SOURCES.md`, then replace `none` with the snapshot URL. These are the sources the collision matrix depends on most and the ones most likely to change. 6. Confirm that no employer, customer, personal, confidential, or credential material is present. -7. Confirm that `VERSION`, `CITATION.cff`, `CHANGELOG.md`, and the release tag agree with each other. Never re-tag existing content with an older version: the prior-art claim is dated, and a release tag that back-dates it corrupts the only thing this repository is for. +7. Confirm that `VERSION`, `CITATION.cff`, `CHANGELOG.md`, and the release tag agree with each other. Run `python3 scripts/check_publication.py --tag v$(cat VERSION)` after creating the local annotated tag and before pushing it. Never re-tag existing content with an older version: the prior-art claim is dated, and a release tag that back-dates it corrupts the only thing this repository is for. 8. Enable GitHub Issues. 9. Enable GitHub private vulnerability reporting before pointing readers to `SECURITY.md`. 10. Decide whether GitHub Discussions should be enabled for design debate; keep factual corrections and prior-art challenges in Issues so they remain traceable. @@ -32,6 +39,11 @@ Do not call the repository a standard, certified protocol, proven deletion syste 15. Create the `prior-art`, `correction`, `conformance`, `implementation`, and `maintenance` labels used by the issue forms and Dependabot. 16. Publish [REVIEW_REQUEST.md](REVIEW_REQUEST.md), [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md), and [PHASE3_SYSTEMS.md](PHASE3_SYSTEMS.md) only as calls for evidence. Record an external review, independent implementation, or system experiment in the readiness ledger only after its dated, independently produced result exists. +The default publication checker is deliberately offline. It validates tracked +metadata and local Git bindings; it does not prove GitHub comments still exist +or remain unchanged. Live publication claims require a fresh repository-wide +GitHub inventory and schema-v2 freshness receipt after the last mutation. + ## Experimental release policy Publishing an experimental version and declaring production readiness are diff --git a/README.md b/README.md index c36643e..bae1fcb 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,10 @@ | Field | Value | |---|---| | Author | Thomas Rainer Willner | -| Version | 0.4.0 | +| Version | 0.4.1 | | Status | Public concept proposal / Request for Comment | | Published | 2026-08-07 | -| Latest release | v0.4.0 — first materially improved experimental release | +| Latest release | v0.4.1 — integrity and conformance hardening release | | Research reviewed through | 2026-08-01 | | License | Attributed specification implementations permitted; reference code remains personal-use. See [LICENSE](LICENSE). | @@ -148,6 +148,11 @@ make links # liveness of every cited external URL (needs network) | `make readiness` | Is the recorded readiness evidence structurally honest and synchronized with the human matrix? | The ledger is malformed, evidence is insufficient for a recorded status, or the matrix contradicts the ledger. | | `make test` | Do those checkers reject what they claim to reject? | A checker has stopped catching a fault it is supposed to catch. | +`make publication` proves offline manifest consistency and local Git binding +only. It prints remote GitHub state as unverified; a live publication or latest +feedback claim additionally requires the repository-wide freshness receipt +described in `PUBLISHING.md`. + The self-tests exist because a check that has never failed has not been shown to work. The self-tests build corpora that misstate the proposal — an inverted quarantine ordering, an asserted world first — and require the guard to reject @@ -180,12 +185,12 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.4.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface include conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, mutation coverage for every signed receipt field, and independently authored adapter-level cases with target-instance tracing, complete outcomes, bounded proposition multiplicity, exact semantic mutations, and executable validator anti-vacuity controls. G2 remains `BLOCKED`: interested-party findings and internal remediation do not replace a complete independent review of the current surface. G4 also remains `BLOCKED`: one externally authored adapter candidate exists, but two independent implementations and a separately produced third-party validator result are not established. +Version 0.4.1 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. It hardens immutable review/release binding, publication-check claims, readiness matrix equality and external-evidence independence, full-width state roots, snapshot limitation disclosure, licence scope, and conformance-corpus digest behavior. Phase 1 and the internal Phase 2 conformance surface retain conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. G2 remains `BLOCKED`: interested-party findings and internal remediation do not replace a complete independent review of the current surface. G4 also remains `BLOCKED`: one externally authored adapter candidate exists, but two independent implementations and a separately produced third-party validator result are not established. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. Experimental release readiness is separate from production readiness. Version -0.4.0 is published so implementers can evaluate and extend a materially better +0.4.1 is published so implementers can evaluate and extend a materially better baseline while G2–G6 remain an explicit backlog. External challenge is welcome whenever users or reviewers encounter the project, but no release claims those gates passed merely because a reviewer did not appear. diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md index f02d06f..5802fc8 100644 --- a/REVIEW_REQUEST.md +++ b/REVIEW_REQUEST.md @@ -8,6 +8,15 @@ Repository: https://github.com/thomaswillner/llm-errata Current verdict: **NOT_PROD_READY**. +## Immutable target model + +The `review_target` in `publication/active-surfaces.json` and the +`normative_target` in `spec/adapter-conformance.json` must be identical. That +commit is the normative source target. The final release commit may be a later +metadata-only packaging commit; its tag is validated separately. Reviewers +should report both identities and must not substitute the release tag for the +normative target without checking the allowed packaging delta. + ## Reviews requested ### Novelty and conformance review diff --git a/SECURITY.md b/SECURITY.md index 8c89ef0..4d91e74 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,7 +5,7 @@ LLM Errata is currently a public concept proposal and may later include schemas, ## Supported versions Until a later policy states otherwise, only the latest versioned release is -eligible for security fixes. Support moved from 0.3.x to 0.4.x with the immutable `v0.4.0` release. Development revisions after the latest release +eligible for security fixes. The current immutable supported release is `v0.4.1`. Development revisions after the latest release receive fixes at maintainer discretion and are not represented as supported releases. @@ -51,6 +51,12 @@ Security-relevant findings include, but are not limited to: - leakage of memory contents, provenance, or deletion requests; - unsafe reference code or conformance tooling added to the repository. +`errata adapter-conformance --binding` executes imported Python and is for +**trusted code only**. Its source-identity and timeout controls are evidence +bindings, not sandboxing. Untrusted candidate code requires external process or +container isolation with no ambient secrets and explicit filesystem, network, +CPU, and memory limits. + Factual disagreements, prior-art reports, and specification design proposals are not vulnerabilities; submit them through the normal contribution process. Licence attribution does not imply security review, endorsement, or certification diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 3705a5b..0c3f652 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -19,6 +19,11 @@ mitigation or an admitted limit. ## Defended +Receipt pre/post state roots use full-width SHA-256 (64 lowercase hexadecimal +characters). Their scope remains limited to inspectable ledger and adapter +snapshots. Any adapter without `snapshot()` contributes no state and must carry +an explicit signed limitation that its mutations are not bound by those roots. + | Threat | Mitigation | Where | |---|---|---| | Forged erratum | Ed25519 over a canonical serialisation; the feed is refused, not the entry | `errata.verify_feed` | diff --git a/VERSION b/VERSION index 1d0ba9e..267577d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0 +0.4.1 diff --git a/prototype/README.md b/prototype/README.md index b206869..2bd4c21 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -143,6 +143,11 @@ without creating a workspace. The default reference binding drives the real `Importer` lifecycle through a proxy around the exact adapter instance. A third-party binding is supplied as `--binding module:factory`. +That binding is **trusted code only**: it executes in the invoking Python +process. Clean-tree checks, admitted-byte loading, namespace isolation, and +timeouts bind evidence but do not provide an operating-system sandbox. Run +untrusted candidates only in a separately constrained process or container. + The canonical JSON report separates the immutable normative predecessor target from the runtime commit being exercised, lists complete honest and mutation outcomes, records exact target-instance calls, and includes three executable diff --git a/prototype/checkpoints.py b/prototype/checkpoints.py index 36a9dd8..dbd22db 100644 --- a/prototype/checkpoints.py +++ b/prototype/checkpoints.py @@ -112,7 +112,9 @@ def __post_init__(self) -> None: raise CheckpointError("checkpoint sequence must be a positive integer") if not isinstance(self.target_root, str) or not self.target_root: raise CheckpointError("checkpoint target root is invalid") - if not isinstance(self.pre_state_root, str) or not self.pre_state_root: + if not isinstance(self.pre_state_root, str) or not _DIGEST.fullmatch( + self.pre_state_root + ): raise CheckpointError("checkpoint pre-state root is invalid") if not isinstance(self.adapters, tuple) or not self.adapters: raise CheckpointError("checkpoint adapters must be non-empty") diff --git a/prototype/conformance.py b/prototype/conformance.py index 4d03190..2f8fca5 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -261,8 +261,6 @@ def canonical_json(self) -> str: RECEIPT_KEYS = {"names_store", "non_trivial", "forbidden_absent"} MUTATION_KEYS = {"id", "exact_counter_result"} CONTROL_KEYS = {"id", "mutation", "required_failure"} -REQUIRED_TARGET = "ac4468faf73c2cc7949dd29b2a2a151f5bd23116" -REQUIRED_DIGEST = "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" GIT_TIMEOUT_SECONDS = 10.0 BINDING_TIMEOUT_SECONDS = 10.0 REQUIRED_PROVENANCE = { @@ -330,6 +328,7 @@ def _surface_paths_at_commit(root: Path, commit: str) -> tuple[str, ...]: groups = ( tuple(sorted(path for path in files if re.fullmatch(r"prototype/[^/]+\.py", path))), ("prototype/README.md", "spec/README.md"), + ("spec/adapter-conformance.json",), tuple(sorted(path for path in files if re.fullmatch(r"spec/[^/]+\.schema\.json", path))), tuple(sorted(path for path in files if re.fullmatch(r"spec/vectors/[^/]+\.json", path))), tuple(sorted(path for path in files if re.fullmatch(r"spec/semantic/[^/]+\.json", path))), @@ -345,9 +344,25 @@ def _surface_paths_at_commit(root: Path, commit: str) -> tuple[str, ...]: def _surface_digest_at_commit(root: Path, commit: str) -> str: digest = hashlib.sha256() for relative in _surface_paths_at_commit(root, commit): + content = _git(root, "show", f"{commit}:{relative}") + if relative == "spec/adapter-conformance.json": + try: + payload = json.loads(content.decode("utf-8")) + target = payload["normative_target"] + if not isinstance(target, dict): + raise TypeError + target["commit"] = "0" * 40 + target["surface_digest"] = "0" * 64 + content = json.dumps( + payload, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + except (UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError) as error: + raise ConformanceInputError( + "conformance corpus target metadata is malformed" + ) from error digest.update(relative.encode("utf-8")) digest.update(b"\0") - digest.update(_git(root, "show", f"{commit}:{relative}")) + digest.update(content) digest.update(b"\0") return digest.hexdigest() @@ -369,6 +384,18 @@ def _validate_outcome(value: object, operation: str, label: str) -> dict[str, An return outcome +def _publication_target(source_root: Path) -> dict[str, Any]: + try: + publication = json.loads( + (source_root / "publication" / "active-surfaces.json").read_text( + encoding="utf-8" + ) + ) + return _exact(publication["review_target"], TARGET_KEYS, "publication target") + except (OSError, UnicodeError, json.JSONDecodeError, KeyError) as error: + raise ConformanceInputError("publication review target is unavailable") from error + + def load_corpus( path: Path, source_root: Path, *, require_canonical_path: bool = False ) -> AdapterCorpus: @@ -385,10 +412,10 @@ def load_corpus( if root["schema_version"] != 1: raise ConformanceInputError("corpus schema version must be 1") target = _exact(root["normative_target"], TARGET_KEYS, "normative target") - if target["commit"] != REQUIRED_TARGET: - raise ConformanceInputError("normative target commit is not canonical") - if target["surface_digest"] != REQUIRED_DIGEST: - raise ConformanceInputError("normative surface digest is not canonical") + if target != _publication_target(source_root): + raise ConformanceInputError( + "normative target does not match the active publication review target" + ) actual_digest = _surface_digest_at_commit(source_root, target["commit"]) if actual_digest != target["surface_digest"]: raise ConformanceInputError("normative surface digest does not match source") diff --git a/prototype/controller.py b/prototype/controller.py index 52d3935..60c0fe0 100644 --- a/prototype/controller.py +++ b/prototype/controller.py @@ -269,7 +269,7 @@ def _validate_checkpoint( if ( record.artifact_ids or record.coverage != "unknown" - or record.limitation != self._opaque_limitation(name) + or record.limitation != self._opaque_limitation(name, adapter) ): raise CheckpointError(f"checkpoint opaque coverage drifted: {name}") continue @@ -288,11 +288,17 @@ def _validate_checkpoint( raise CheckpointError(f"checkpoint artifact is no longer gated: {name}") @staticmethod - def _opaque_limitation(name: str) -> str: - return ( + def _opaque_limitation(name: str, adapter: StoreAdapter | None = None) -> str: + limitation = ( f"{name}: store exposes no enumeration interface, so its coverage " "is unknown and no repair elsewhere changes that" ) + if adapter is not None and not callable(getattr(adapter, "snapshot", None)): + limitation += ( + "; adapter exposes no state snapshot, so checkpoint and receipt " + "state roots cannot bind its mutations" + ) + return limitation @staticmethod def _lineage_limitation(adapter: StoreAdapter, root: str) -> str | None: @@ -310,18 +316,24 @@ def _lineage_limitation(adapter: StoreAdapter, root: str) -> str | None: except Exception: complete = False snapshot = getattr(adapter, "snapshot", None) - if complete and callable(snapshot): - return None - if complete: - return ( + snapshot_limitation = None + if not callable(snapshot): + snapshot_limitation = ( f"{adapter.name}: adapter exposes no state snapshot for {root}; " "checkpoint and receipt state roots cannot bind its mutations" ) - return ( + if complete and snapshot_limitation is None: + return None + if complete: + return snapshot_limitation + limitation = ( f"{adapter.name}: enumeration returned a result but the adapter did " f"not establish complete root-specific lineage for {root}; empty or " "partial walks cannot become verified coverage" ) + if snapshot_limitation is not None: + limitation += f"; {snapshot_limitation}" + return limitation @staticmethod def _feed_view_limitation() -> str: @@ -374,7 +386,9 @@ def _quarantine( acknowledge = getattr(adapter, "acknowledge", None) if acknowledge is not None: acknowledge(root) - limitations[adapter.name] = self._opaque_limitation(adapter.name) + limitations[adapter.name] = self._opaque_limitation( + adapter.name, adapter + ) gated[adapter.name] = [] checkpoint_coverage[adapter.name] = Coverage.UNKNOWN continue diff --git a/prototype/signing.py b/prototype/signing.py index 435ce5e..affe8e8 100644 --- a/prototype/signing.py +++ b/prototype/signing.py @@ -43,7 +43,7 @@ def canonical_bytes(payload: dict[str, Any]) -> bytes: def commitment(*parts: str) -> str: - """A short, stable digest used to bind states and identify artifacts. + """A full-width SHA-256 digest used to bind inspectable importer state. This is never applied to an erased value. Committing to a low-entropy proposition — "vegetarian" — would produce a digest an attacker can confirm @@ -56,7 +56,7 @@ def commitment(*parts: str) -> str: for part in parts: digest.update(part.encode("utf-8")) digest.update(b"\x1f") - return digest.hexdigest()[:32] + return digest.hexdigest() @dataclass(frozen=True) diff --git a/publication/active-surfaces.json b/publication/active-surfaces.json index efea6b8..d9d270e 100644 --- a/publication/active-surfaces.json +++ b/publication/active-surfaces.json @@ -1,9 +1,18 @@ { - "schema_version": 2, + "schema_version": 3, "review_target": { "commit": "ad36ed5e209a53aacab17751b5a183ca8a1aac1f", "surface_digest": "3ca427bb2645517e1b1d921859a7721896644a05e32ea39fb0071a021ddc5b6d" }, + "release_binding": { + "version": "0.4.1", + "tag": "v0.4.1", + "allowed_packaging_paths": [ + "publication/active-surfaces.json", + "spec/adapter-conformance.json" + ], + "commit_model": "review target plus metadata-only packaging commit; tag binds release commit" + }, "license": { "specification_implementation": "irrevocable worldwide royalty-free commercial and non-commercial", "required_attribution": [ diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index d63a65c..bb10731 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "project_version": "0.4.0", + "project_version": "0.4.1", "verdict": "NOT_PROD_READY", "last_reviewed": "2026-08-14", "gates": [ diff --git a/scripts/check_publication.py b/scripts/check_publication.py index 52f744d..e1511fb 100644 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -1,305 +1,269 @@ #!/usr/bin/env python3 -"""Validate the tracked active-publication-surface manifest offline.""" +"""Validate local publication metadata without pretending to verify GitHub.""" from __future__ import annotations import json import re +import subprocess import sys from datetime import date from pathlib import Path from urllib.parse import urlparse +from check_readiness import g2_surface_digest, g2_surface_digest_at_commit + ROOT = Path(__file__).resolve().parents[1] MANIFEST = ROOT / "publication" / "active-surfaces.json" - -CANONICAL_COMMIT = "ad36ed5e209a53aacab17751b5a183ca8a1aac1f" -CANONICAL_DIGEST = ( - "3ca427bb2645517e1b1d921859a7721896644a05e32ea39fb0071a021ddc5b6d" -) +CORPUS = ROOT / "spec" / "adapter-conformance.json" REPOSITORY_URL = "https://github.com/thomaswillner/llm-errata" ALLOWED_GATES = {"G2", "G3", "G4", "G5", "G6"} REQUIRED_ATTRIBUTION = {"LLM Errata", "Thomas Willner", REPOSITORY_URL} -REQUIRED_SURFACES = { - "g4-inspeximus-v040-target-reply": { - "kind": "issue-comment", - "url": f"{REPOSITORY_URL}/issues/4#issuecomment-5287579823", - "gates": ["G2", "G4"], - "roles": ["inspeximus-adapter-author"], - "mentions": ["DanceNitra"], - "evidence_boundary": "recruitment-only", - }, +ALLOWED_PACKAGING_PATHS = { + "publication/active-surfaces.json", + "spec/adapter-conformance.json", } ROOT_KEYS = { - "schema_version", "review_target", "license", "historical_surfaces", "surfaces" + "schema_version", "review_target", "release_binding", "license", + "historical_surfaces", "surfaces", } TARGET_KEYS = {"commit", "surface_digest"} +RELEASE_KEYS = {"version", "tag", "allowed_packaging_paths", "commit_model"} LICENSE_KEYS = { - "specification_implementation", - "required_attribution", - "reference_code", + "specification_implementation", "required_attribution", "reference_code", "case_by_case_permission_required", } SURFACE_KEYS = { - "id", - "kind", - "url", - "published", - "commit", - "surface_digest", - "gates", - "roles", - "mentions", - "supersedes", - "evidence_boundary", + "id", "kind", "url", "published", "commit", "surface_digest", "gates", + "roles", "mentions", "supersedes", "evidence_boundary", } HISTORICAL_SURFACE_KEYS = SURFACE_KEYS | {"superseded_by"} -REQUIRED_HISTORICAL_SURFACES = [ - { - "id": "g4-inspeximus-current-target-reply", - "kind": "issue-comment", - "url": f"{REPOSITORY_URL}/issues/4#issuecomment-5282207719", - "published": "2026-08-13", - "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", - "surface_digest": ( - "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" - ), - "gates": ["G2", "G4"], - "roles": ["inspeximus-adapter-author"], - "mentions": ["DanceNitra"], - "supersedes": [ - f"{REPOSITORY_URL}/issues/4#issuecomment-5280210050", - f"{REPOSITORY_URL}/pull/8#issuecomment-5280225709", - ], - "evidence_boundary": "recruitment-only", - "superseded_by": f"{REPOSITORY_URL}/issues/4#issuecomment-5287579823", - } -] def load_manifest(path: Path) -> object: return json.loads(path.read_text(encoding="utf-8")) +def _is_repository_url(value: object) -> bool: + if not isinstance(value, str): + return False + parsed = urlparse(value) + return ( + parsed.scheme == "https" + and parsed.netloc == "github.com" + and parsed.path.startswith("/thomaswillner/llm-errata/") + and not parsed.username + and not parsed.password + ) + + +def _valid_date(value: object) -> bool: + if not isinstance(value, str) or re.fullmatch(r"\d{4}-\d{2}-\d{2}", value) is None: + return False + try: + return date.fromisoformat(value) <= date.today() + except ValueError: + return False + + +def _valid_target(value: object) -> bool: + return ( + isinstance(value, dict) + and set(value) == TARGET_KEYS + and isinstance(value.get("commit"), str) + and re.fullmatch(r"[0-9a-f]{40}", value["commit"]) is not None + and isinstance(value.get("surface_digest"), str) + and re.fullmatch(r"[0-9a-f]{64}", value["surface_digest"]) is not None + ) + + +def _validate_surface( + surface: object, + *, + target: dict[str, str], + historical: bool, +) -> list[str]: + failures: list[str] = [] + expected_keys = HISTORICAL_SURFACE_KEYS if historical else SURFACE_KEYS + if not isinstance(surface, dict) or set(surface) != expected_keys: + return ["surface fields: exact versioned fields are required"] + values = (surface.get("gates"), surface.get("roles"), surface.get("mentions"), surface.get("supersedes")) + valid_lists = all( + isinstance(value, list) + and all(isinstance(item, str) and bool(item) for item in value) + for value in values + ) + boundary = surface.get("evidence_boundary") + if not ( + isinstance(surface.get("id"), str) + and re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", surface["id"]) is not None + and surface.get("kind") in {"issue-comment", "pull-request-comment", "discussion-comment"} + and _is_repository_url(surface.get("url")) + and _valid_date(surface.get("published")) + and isinstance(surface.get("commit"), str) + and re.fullmatch(r"[0-9a-f]{40}", surface["commit"]) is not None + and isinstance(surface.get("surface_digest"), str) + and re.fullmatch(r"[0-9a-f]{64}", surface["surface_digest"]) is not None + and valid_lists + and bool(surface["gates"]) + and len(surface["gates"]) == len(set(surface["gates"])) + and set(surface["gates"]).issubset(ALLOWED_GATES) + and len(surface["supersedes"]) == len(set(surface["supersedes"])) + and all(_is_repository_url(item) for item in surface["supersedes"]) + and boundary in {"recruitment-only", "publication-only"} + ): + failures.append("surface fields: invalid ID, URL, date, target, gates, supersession, or boundary") + if not historical and ( + surface.get("commit") != target["commit"] + or surface.get("surface_digest") != target["surface_digest"] + ): + failures.append("surface target binding: active surfaces must bind the review target") + if boundary == "recruitment-only" and not surface.get("roles"): + failures.append("evidence boundary: recruitment surfaces require roles") + if boundary == "publication-only" and (surface.get("roles") or surface.get("mentions")): + failures.append("evidence boundary: publication surfaces cannot recruit or mention users") + if historical and not _is_repository_url(surface.get("superseded_by")): + failures.append("historical surfaces: superseded_by must be a repository URL") + return failures + + +def _git(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], cwd=ROOT, capture_output=True, text=True, check=False + ) + + def validate_manifest(payload: object) -> list[str]: failures: list[str] = [] if not isinstance(payload, dict) or set(payload) != ROOT_KEYS: return ["manifest schema: root must contain the exact required fields"] - - if payload.get("schema_version") != 2: - failures.append("manifest schema: schema_version must be 2") + if payload.get("schema_version") != 3: + failures.append("manifest schema: schema_version must be 3") target = payload.get("review_target") - target_valid = ( - isinstance(target, dict) - and set(target) == TARGET_KEYS - and target.get("commit") == CANONICAL_COMMIT - and target.get("surface_digest") == CANONICAL_DIGEST - ) - if not target_valid: - failures.append( - "review target: commit and digest must equal the corrected canonical target" - ) + if not _valid_target(target): + failures.append("review target: full commit and SHA-256 surface digest are required") + target = {"commit": "", "surface_digest": ""} + + release = payload.get("release_binding") + version = (ROOT / "VERSION").read_text(encoding="utf-8").strip() + if not ( + isinstance(release, dict) + and set(release) == RELEASE_KEYS + and release.get("version") == version + and release.get("tag") == f"v{version}" + and release.get("allowed_packaging_paths") == sorted(ALLOWED_PACKAGING_PATHS) + and release.get("commit_model") + == "review target plus metadata-only packaging commit; tag binds release commit" + ): + failures.append("release binding: version, tag, commit model, and packaging paths must align") + + try: + corpus = json.loads(CORPUS.read_text(encoding="utf-8")) + corpus_target = corpus["normative_target"] + except (OSError, UnicodeError, json.JSONDecodeError, KeyError): + corpus_target = None + if target != corpus_target: + failures.append("review target: publication and conformance targets must be identical") license_data = payload.get("license") if not isinstance(license_data, dict) or set(license_data) != LICENSE_KEYS: failures.append("licence posture: exact licence fields are required") else: posture = license_data.get("specification_implementation") - posture_terms = { - "irrevocable", - "worldwide", - "royalty-free", - "commercial", - "non-commercial", - } - posture_valid = ( + terms = {"irrevocable", "worldwide", "royalty-free", "commercial", "non-commercial"} + if not ( isinstance(posture, str) - and all(term in posture.casefold() for term in posture_terms) + and all(term in posture.casefold() for term in terms) and license_data.get("case_by_case_permission_required") is False and "written" not in posture.casefold() and "permission" not in posture.casefold() - ) - if not posture_valid: - failures.append( - "licence posture: attributed independent implementation grant must not require case-by-case permission" - ) - + ): + failures.append("licence posture: implementation grant must not require case-by-case permission") attribution = license_data.get("required_attribution") - if ( - not isinstance(attribution, list) - or any(not isinstance(item, str) for item in attribution) - or set(attribution) != REQUIRED_ATTRIBUTION - or len(attribution) != len(REQUIRED_ATTRIBUTION) + if not ( + isinstance(attribution, list) + and all(isinstance(item, str) for item in attribution) + and set(attribution) == REQUIRED_ATTRIBUTION + and len(attribution) == len(REQUIRED_ATTRIBUTION) ): - failures.append( - "licence attribution: LLM Errata, Thomas Willner, and repository URL are required exactly once" - ) - + failures.append("licence attribution: project, author, and repository are required exactly once") reference_code = license_data.get("reference_code") if not ( isinstance(reference_code, str) and "personal-use" in reference_code.casefold() - and all( - path in reference_code - for path in ("prototype/", "scripts/", "tests/") - ) + and all(path in reference_code for path in ("prototype/", "scripts/", "tests/")) ): - failures.append( - "reference code boundary: personal-use prototype/, scripts/, and tests/ scope is required" - ) - - historical = payload.get("historical_surfaces") - if historical != REQUIRED_HISTORICAL_SURFACES: - failures.append( - "historical surfaces: exact immutable superseded records are required" - ) + failures.append("reference code boundary: personal-use code scope is required") surfaces = payload.get("surfaces") - if not isinstance(surfaces, list): - failures.append("required active surfaces: surfaces must be a list") - return failures - - surface_ids: list[str] = [] - surface_urls: list[str] = [] - roles: list[str] = [] - mentions: list[str] = [] - for index, surface in enumerate(surfaces): - label = f"surface fields: entry {index + 1}" - if not isinstance(surface, dict) or set(surface) != SURFACE_KEYS: - failures.append(f"{label} must contain the exact required fields") - continue - - surface_id = surface.get("id") - kind = surface.get("kind") - url = surface.get("url") - published = surface.get("published") - gates = surface.get("gates") - entry_roles = surface.get("roles") - entry_mentions = surface.get("mentions") - supersedes = surface.get("supersedes") - boundary = surface.get("evidence_boundary") - - expected = REQUIRED_SURFACES.get(surface_id) if isinstance(surface_id, str) else None - valid_date = False - if isinstance(published, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", published): - try: - valid_date = date.fromisoformat(published) <= date.today() - except ValueError: - valid_date = False - valid_url = isinstance(url, str) and _is_repository_url(url) - valid_string_list_fields = all( - isinstance(value, list) - and all(isinstance(item, str) and item for item in value) - for value in (gates, entry_roles, entry_mentions, supersedes) - ) - valid_gates = ( - isinstance(gates, list) - and bool(gates) - and len(gates) == len(set(gates)) - and set(gates).issubset(ALLOWED_GATES) - ) - valid_supersedes = ( - isinstance(supersedes, list) - and bool(supersedes) - and len(supersedes) == len(set(supersedes)) - and all(_is_repository_url(item) for item in supersedes) - ) - expected_fields_match = expected is not None and all( - surface.get(field) == value for field, value in expected.items() - ) - if not ( - isinstance(surface_id, str) - and re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", surface_id) - and kind in {"issue-comment", "pull-request-comment", "discussion-comment"} - and valid_url - and valid_date - and valid_string_list_fields - and valid_gates - and valid_supersedes - and boundary in {"recruitment-only", "publication-only"} - and expected_fields_match - ): - failures.append( - f"{label} has invalid or non-canonical ID, kind, URL, date, gates, roles, mentions, supersession, or boundary" - ) - - if ( - surface.get("commit") != CANONICAL_COMMIT - or surface.get("surface_digest") != CANONICAL_DIGEST - ): - failures.append( - f"surface target binding: {surface_id!r} must bind canonical commit and digest" - ) - - if boundary == "recruitment-only" and not entry_roles: - failures.append( - f"evidence boundary: recruitment surface {surface_id!r} requires roles" - ) - if boundary == "publication-only" and (entry_roles or entry_mentions): - failures.append( - f"evidence boundary: publication surface {surface_id!r} cannot recruit or mention users" - ) - if boundary not in {"recruitment-only", "publication-only"}: - failures.append( - f"evidence boundary: {surface_id!r} cannot represent invitation or publication as independent evidence" - ) - - if isinstance(surface_id, str): - surface_ids.append(surface_id) - if isinstance(url, str): - surface_urls.append(url) - if isinstance(entry_roles, list): - roles.extend(item for item in entry_roles if isinstance(item, str)) - if isinstance(entry_mentions, list): - mentions.extend(item.casefold() for item in entry_mentions if isinstance(item, str)) - - if set(surface_ids) != set(REQUIRED_SURFACES) or len(surface_ids) != len(REQUIRED_SURFACES): - failures.append( - "required active surfaces: manifest must contain each canonical active surface exactly once" - ) - if len(surface_urls) != len(set(surface_urls)): - failures.append("unique surface URLs: active surface URLs must not repeat") + historical = payload.get("historical_surfaces") + if not isinstance(surfaces, list) or not surfaces: + failures.append("required active surfaces: at least one current surface is required") + surfaces = [] + if not isinstance(historical, list) or not historical: + failures.append("historical surfaces: append-only superseded records are required") + historical = [] + for surface in surfaces: + failures.extend(_validate_surface(surface, target=target, historical=False)) + for surface in historical: + failures.extend(_validate_surface(surface, target=target, historical=True)) + + active_ids = [surface.get("id") for surface in surfaces if isinstance(surface, dict)] + active_urls = [surface.get("url") for surface in surfaces if isinstance(surface, dict)] + roles = [role for surface in surfaces if isinstance(surface, dict) for role in surface.get("roles", [])] + mentions = [mention.casefold() for surface in surfaces if isinstance(surface, dict) for mention in surface.get("mentions", [])] + if len(active_ids) != len(set(active_ids)) or len(active_urls) != len(set(active_urls)): + failures.append("unique surface URLs: active IDs and URLs must not repeat") if len(roles) != len(set(roles)): - failures.append("unique evidence roles: each evidence role needs one owner") + failures.append("unique evidence roles: each active role needs one owner") if len(mentions) != len(set(mentions)): failures.append("unique GitHub mentions: each identity may be notified only once") - active_url_set = set(surface_urls) - for surface in surfaces: - if isinstance(surface, dict) and isinstance(surface.get("supersedes"), list): - if active_url_set.intersection(surface["supersedes"]): - failures.append( - "surface fields: active surface URL cannot also be superseded" - ) - break - - if isinstance(historical, list): - historical_urls = { - item.get("url") for item in historical if isinstance(item, dict) - } - if active_url_set.intersection(historical_urls): - failures.append("historical surfaces: historical URL cannot remain active") - + if (ROOT / ".git").exists() and _valid_target(target): + commit = target["commit"] + ancestor = _git("merge-base", "--is-ancestor", commit, "HEAD") + if ancestor.returncode != 0: + failures.append("review target: commit must be an ancestor of runtime HEAD") + try: + if target["surface_digest"] != g2_surface_digest_at_commit(commit, ROOT): + failures.append("review target: committed surface digest does not match") + if target["surface_digest"] != g2_surface_digest(ROOT): + failures.append("review target: runtime surface differs from reviewed source") + except OSError as error: + failures.append(f"review target: {error}") + changed = _git("diff", "--name-only", f"{commit}..HEAD") + if changed.returncode != 0: + failures.append("release binding: packaging delta cannot be inspected") + elif set(changed.stdout.splitlines()) - ALLOWED_PACKAGING_PATHS: + failures.append("release binding: runtime contains non-packaging changes after review target") return failures -def _is_repository_url(value: object) -> bool: - if not isinstance(value, str): - return False - parsed = urlparse(value) - return ( - parsed.scheme == "https" - and parsed.netloc == "github.com" - and parsed.path.startswith("/thomaswillner/llm-errata/") - and not parsed.username - and not parsed.password - ) +def _validate_tag(tag: str) -> tuple[int, str]: + if not (ROOT / ".git").exists(): + return 2, "[INCONCLUSIVE] release tag: Git metadata is unavailable" + resolved = _git("rev-parse", f"refs/tags/{tag}^{{commit}}") + if resolved.returncode != 0: + return 2, f"[INCONCLUSIVE] release tag: {tag} does not exist" + head = _git("rev-parse", "HEAD") + if resolved.stdout.strip() != head.stdout.strip(): + return 1, f"[FAIL] release tag: {tag} does not resolve to HEAD" + return 0, f"[PASS] release tag: {tag} resolves to current release commit" def main() -> int: + args = sys.argv[1:] + require_remote = "--require-remote" in args + tag = None + if "--tag" in args: + index = args.index("--tag") + if index + 1 >= len(args): + print("[FAIL] release tag: --tag requires a value") + return 1 + tag = args[index + 1] try: payload = load_manifest(MANIFEST) except (OSError, UnicodeError, json.JSONDecodeError) as exc: @@ -313,13 +277,19 @@ def main() -> int: print(f"\nPublication validation failed: {len(failures)} issue(s).") return 1 - print("[PASS] manifest schema: exact versioned fields") - print("[PASS] review target: corrected immutable commit and digest") - print("[PASS] licence posture: implementation rights and attribution aligned") - print("[PASS] active surfaces: required URLs, roles, mentions, and boundaries") - print("\nPublication validation passed.") + print("[PASS] offline manifest consistency: local schema, target, licence, and evidence boundaries align") + print("[UNVERIFIED] remote GitHub surfaces were not verified by this offline checker") + if tag is not None: + status, message = _validate_tag(tag) + print(message) + if status: + return status + if require_remote: + print("[INCONCLUSIVE] remote publication state requires a fresh GitHub inventory receipt") + return 2 + print("\nOffline publication metadata validation passed; remote state remains unverified.") return 0 if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 0464da0..9d92fc2 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -30,7 +30,7 @@ STATUSES = {"PASS", "FAIL", "BLOCKED"} CLASSES = {"internal", "external"} GENERIC_NON_INDEPENDENT_PRODUCER_RE = re.compile( - r"(?:^|[\s:_-])(local|self|maintainer|agent|repository|repo)(?:$|[\s:_-])", + r"(?:^|[\s:_-])(author|owner|implementer|operator|contributor|maintainer|thomas|willner|project|reference|local|agent|repository|repo|self)(?:$|[\s:_-])", re.IGNORECASE, ) G2_NON_INDEPENDENT_PRODUCER_RE = re.compile( @@ -64,6 +64,13 @@ } ) G2_RESULTS = {"pass", "pass-with-findings", "fail"} +G3_ATTESTATION = "llm-errata-independent-cryptography-review-v1" +G3_SCOPE = frozenset({ + "library-build", "constant-time", "malformed-input-refusal", + "key-rotation", "key-recovery", "revocation", "delegation", +}) +G4_ATTESTATION = "llm-errata-independent-implementation-v1" +G5_ATTESTATION = "llm-errata-independent-interoperability-review-v1" G6_ATTESTATION = "llm-errata-independent-operational-review-v1" G6_SCOPE = frozenset( { @@ -214,9 +221,35 @@ def surface_digest_from_bytes(entries: list[tuple[str, bytes]]) -> str: return digest.hexdigest() +def review_surface_content(relative: str, content: bytes) -> bytes: + """Normalize only the corpus's self-referential target pointer. + + Cases, provenance, controls, and every other corpus byte remain digest-bound. + The target commit and digest are validated separately. Normalizing those two + fields lets a later packaging commit point at the immutable source commit + without requiring a Git commit to contain its own hash. + """ + + if relative != "spec/adapter-conformance.json": + return content + try: + payload = json.loads(content.decode("utf-8")) + target = payload["normative_target"] + if not isinstance(target, dict): + raise TypeError + target["commit"] = "0" * 40 + target["surface_digest"] = "0" * 64 + except (UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError) as error: + raise OSError("conformance corpus target metadata is malformed") from error + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + + def g2_surface_digest(root: Path = ROOT) -> str: return surface_digest_from_bytes( - [(relative, (root / relative).read_bytes()) for relative in g2_surface_files(root)] + [ + (relative, review_surface_content(relative, (root / relative).read_bytes())) + for relative in g2_surface_files(root) + ] ) @@ -236,13 +269,13 @@ def g2_surface_digest_at_commit(commit: str, root: Path = ROOT) -> str: entries = [] for relative in g2_surface_files(root): if relative == "spec/adapter-conformance.json" and relative not in commit_files: - continue + raise OSError("reviewed commit predates the conformance corpus") result = subprocess.run( ["git", "show", f"{commit}:{relative}"], cwd=root, capture_output=True, check=False ) if result.returncode != 0: raise OSError(f"reviewed commit lacks {relative}") - entries.append((relative, result.stdout)) + entries.append((relative, review_surface_content(relative, result.stdout))) return surface_digest_from_bytes(entries) @@ -324,6 +357,189 @@ def qualifying_g2_review_evidence( } +def _surface_digest_for_files(files: tuple[str, ...], root: Path) -> str: + if any(not (root / relative).is_file() for relative in files): + raise OSError("gate surface is incomplete") + return surface_digest_from_bytes( + [(relative, (root / relative).read_bytes()) for relative in sorted(files)] + ) + + +def _surface_digest_for_files_at_commit( + files: tuple[str, ...], commit: str, root: Path +) -> str: + if not reviewed_commit_exists(commit, root): + raise OSError("reviewed commit is unavailable") + entries = [] + for relative in sorted(files): + result = subprocess.run( + ["git", "show", f"{commit}:{relative}"], cwd=root, + capture_output=True, check=False, + ) + if result.returncode != 0: + raise OSError(f"reviewed commit lacks {relative}") + entries.append((relative, result.stdout)) + return surface_digest_from_bytes(entries) + + +G3_SURFACE_FILES = ( + "prototype/ed25519.py", "prototype/errata.py", "prototype/signing.py", + "THREAT_MODEL.md", "docs/CRYPTOGRAPHY_QUALIFICATION.md", + "tests/test_ed25519.py", "tests/test_errata_feed.py", +) +G5_SURFACE_FILES = ( + "PHASE3_SYSTEMS.md", "ROADMAP.md", "spec/erratum.schema.json", + "spec/receipt.schema.json", +) + + +def _commit_bound_external( + entry: object, + *, + attestation: str, + surface_files: tuple[str, ...] | None, + today: date | None, + root: Path, +) -> bool: + if not valid_external_evidence(entry, today=today) or not isinstance(entry, dict): + return False + commit = entry.get("reviewed_commit") + if not ( + entry.get("kind") == "external" + and valid_g2_report_ref(entry.get("ref")) + and isinstance(commit, str) + and re.fullmatch(r"[0-9a-f]{40}", commit) is not None + and entry.get("relationship") in { + "independent-third-party", "independent-implementation", + "independent-third-party-validator", "independent-experiment-report", + } + and isinstance(entry.get("conflicts"), list) + and valid_g2_identity_ref(entry.get("producer_identity")) + and entry.get("independence_attestation") == attestation + and entry.get("result") in G2_RESULTS + ): + return False + try: + if surface_files is None: + current = g2_surface_digest(root) + committed = g2_surface_digest_at_commit(commit, root) + else: + current = _surface_digest_for_files(surface_files, root) + committed = _surface_digest_for_files_at_commit(surface_files, commit, root) + return entry.get("surface_digest") == current == committed + except OSError: + return False + + +def qualifying_g3_security_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: + if not _commit_bound_external( + entry, attestation=G3_ATTESTATION, surface_files=G3_SURFACE_FILES, + today=today, root=root, + ) or not isinstance(entry, dict): + return False + implementation = entry.get("implementation") + return ( + entry.get("review_type") == "production-cryptography" + and entry.get("relationship") == "independent-third-party" + and entry.get("result") in {"pass", "pass-with-findings"} + and isinstance(entry.get("scope"), list) + and set(entry["scope"]) == G3_SCOPE + and len(entry["scope"]) == len(G3_SCOPE) + and _exact_dict(implementation, { + "library", "version", "binding", "build_digest", "platforms", + "constant_time", "audited_build", + }) + and all(_nonempty(implementation[key]) for key in ("library", "version", "binding")) + and isinstance(implementation["build_digest"], str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", implementation["build_digest"]) is not None + and isinstance(implementation["platforms"], list) and bool(implementation["platforms"]) + and all(_nonempty(item) for item in implementation["platforms"]) + and implementation["constant_time"] is True + and implementation["audited_build"] is True + ) + + +def valid_g4_implementation_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: + if not _commit_bound_external( + entry, attestation=G4_ATTESTATION, surface_files=None, + today=today, root=root, + ) or not isinstance(entry, dict): + return False + role = entry.get("evidence_role") + expected_relationship = ( + "independent-implementation" if role == "adapter" + else "independent-third-party-validator" + ) + return ( + entry.get("review_type") == "g4-conformance" + and role in {"adapter", "validator"} + and entry.get("relationship") == expected_relationship + and entry.get("result") in {"pass", "pass-with-findings"} + and _nonempty(entry.get("implementation_id")) + ) + + +def qualifying_g4_evidence( + entries: list[dict[str, object]], *, today: date | None = None, root: Path = ROOT +) -> bool: + valid = [ + entry for entry in entries + if valid_g4_implementation_evidence(entry, today=today, root=root) + ] + adapters = [entry for entry in valid if entry["evidence_role"] == "adapter"] + validators = [entry for entry in valid if entry["evidence_role"] == "validator"] + if len(adapters) < 2 or not validators: + return False + adapter_identities = {entry["producer_identity"] for entry in adapters} + adapter_implementations = {entry["implementation_id"] for entry in adapters} + validator_identities = {entry["producer_identity"] for entry in validators} + targets = {(entry["reviewed_commit"], entry["surface_digest"]) for entry in valid} + return ( + len(adapter_identities) >= 2 + and len(adapter_implementations) >= 2 + and validator_identities.isdisjoint(adapter_identities) + and len(targets) == 1 + ) + + +def qualifying_g5_interoperability_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: + if not _commit_bound_external( + entry, attestation=G5_ATTESTATION, surface_files=G5_SURFACE_FILES, + today=today, root=root, + ) or not isinstance(entry, dict): + return False + systems = entry.get("systems") + if not ( + entry.get("review_type") == "phase3-interoperability" + and entry.get("relationship") == "independent-experiment-report" + and entry.get("result") in {"pass", "pass-with-findings"} + and entry.get("synthetic_data") is True + and _nonempty(entry.get("root_id")) + and isinstance(systems, list) and len(systems) == 3 + ): + return False + for system in systems: + if not ( + _exact_dict(system, { + "name", "version", "operator", "operator_identity", + "evidence_ref", "result", "independently_operated", + }) + and all(_nonempty(system[key]) for key in ("name", "version", "operator")) + and valid_g2_identity_ref(system["operator_identity"]) + and valid_g2_report_ref(system["evidence_ref"]) + and system["result"] == "pass" + and system["independently_operated"] is True + ): + return False + return len({system["operator_identity"] for system in systems}) == 3 + + def g6_surface_files(root: Path = ROOT) -> tuple[str, ...]: relative = ( "docs/OPERATIONAL_READINESS.md", @@ -600,6 +816,28 @@ def validate_matrix( "all rows must be unique, well formed, and exact", ) + ledger_gates = { + gate.get("id"): gate + for gate in raw_gates + if isinstance(gate, dict) and isinstance(gate.get("id"), str) + } if isinstance(raw_gates, list) else {} + matrix_gate_rows = { + markdown_value(row[0]): row + for row in gate_rows + if len(row) == 5 and markdown_value(row[0]) in REQUIRED_GATES + } + for gate_id in sorted(REQUIRED_GATES): + gate = ledger_gates.get(gate_id) + row = matrix_gate_rows.get(gate_id) + criterion = gate.get("criterion") if isinstance(gate, dict) else None + reporter.check( + f"{gate_id} matrix criterion", + isinstance(criterion, str) + and row is not None + and markdown_value(row[1]) == criterion, + f"{gate_id} matrix criterion exactly matches the readiness ledger", + ) + g2_gate = next( (gate for gate in raw_gates if isinstance(gate, dict) and gate.get("id") == "G2"), None, @@ -608,14 +846,6 @@ def validate_matrix( (row for row in gate_rows if len(row) == 5 and markdown_value(row[0]) == "G2"), None, ) - g2_criterion = g2_gate.get("criterion") if isinstance(g2_gate, dict) else None - reporter.check( - "G2 matrix criterion", - isinstance(g2_criterion, str) - and g2_row is not None - and markdown_value(g2_row[1]) == g2_criterion, - "G2 matrix criterion exactly matches the readiness ledger", - ) reporter.check( "G2 matrix current evidence", g2_row is not None and markdown_value(g2_row[3]) == G2_MATRIX_CURRENT_EVIDENCE, @@ -634,14 +864,6 @@ def validate_matrix( (row for row in gate_rows if len(row) == 5 and markdown_value(row[0]) == "G6"), None, ) - g6_criterion = g6_gate.get("criterion") if isinstance(g6_gate, dict) else None - reporter.check( - "G6 matrix criterion", - isinstance(g6_criterion, str) - and g6_row is not None - and markdown_value(g6_row[1]) == g6_criterion, - "G6 matrix criterion exactly matches the readiness ledger", - ) reporter.check( "G6 matrix current evidence", g6_row is not None and markdown_value(g6_row[3]) == G6_MATRIX_CURRENT_EVIDENCE, @@ -749,6 +971,9 @@ def validate_ledger( valid_external_entries = 0 valid_g2_reviews = 0 + valid_g3_reports = 0 + g4_external_entries: list[dict[str, object]] = [] + valid_g5_reports = 0 valid_g6_reports = 0 evidence_valid = isinstance(evidence, list) if isinstance(evidence, list): @@ -826,6 +1051,12 @@ def validate_ledger( valid_external_entries += 1 if gate_id == "G2" and qualifying_g2_review_evidence(entry, root=ROOT): valid_g2_reviews += 1 + if gate_id == "G3" and qualifying_g3_security_evidence(entry, root=ROOT): + valid_g3_reports += 1 + if gate_id == "G4": + g4_external_entries.append(entry) + if gate_id == "G5" and qualifying_g5_interoperability_evidence(entry, root=ROOT): + valid_g5_reports += 1 if gate_id == "G6" and qualifying_g6_operational_evidence(entry, root=ROOT): valid_g6_reports += 1 else: @@ -837,6 +1068,9 @@ def validate_ledger( qualifying_external = ( valid_g2_reviews if gate_id == "G2" + else valid_g3_reports if gate_id == "G3" + else int(qualifying_g4_evidence(g4_external_entries, root=ROOT)) if gate_id == "G4" + else valid_g5_reports if gate_id == "G5" else valid_g6_reports if gate_id == "G6" else valid_external_entries ) @@ -844,7 +1078,7 @@ def validate_ledger( reporter.check( f"{prefix} external PASS evidence", external_pass_valid, - "external evidence: G2 requires complete conformance review; G6 requires complete measured operational report; other external gates require independent evidence", + "external evidence: every external gate requires its gate-specific independent, commit-bound evidence schema", ) if not external_pass_valid: all_pass = False diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index b00349c..3af24d4 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -448,6 +448,12 @@ def check_publication_metadata(reporter: Reporter) -> None: "No patent rights are granted", "Apache License 2.0", "irrevocable", + "HARD_PROBLEMS.md", + "PRIOR_ART.md", + "RESEARCH.md", + "PRODUCTION_READINESS.md", + "INDEPENDENT_IMPLEMENTATION.md", + "PHASE3_SYSTEMS.md", ) reporter.check( "license and notice", @@ -473,6 +479,7 @@ def check_publication_metadata(reporter: Reporter) -> None: "CONTRIBUTING.md": ("independently authored implementation",), "SECURITY.md": ( "Licence attribution does not imply security review, endorsement, or certification", + "trusted code only", ), } public_alignment = all( diff --git a/spec/README.md b/spec/README.md index 83c013a..e6da45e 100644 --- a/spec/README.md +++ b/spec/README.md @@ -25,6 +25,15 @@ requires calls through the exact adapter instance, and names one flattering mutation with its exact counter-result. An exception is not evidence that a mutation was caught. +The G2 surface digest includes the complete adapter corpus. To avoid a +self-referential Git hash, digest calculation canonicalizes only the corpus's +`normative_target.commit` and `normative_target.surface_digest` values to fixed +zero sentinels; every case, quotation, provenance field, expected outcome, +mutation, control, path, and other byte remains bound. The normalized target +values are validated separately against publication metadata and the immutable +commit. A commit that predates the corpus fails explicitly rather than silently +changing the digested file set. + Preservation includes bounded proposition multiplicity. Within an inspectable synthetic conformance scope, an adapter supplies stable provider-local proposition identities and active assertion counts. Repair must not increase diff --git a/spec/receipt.schema.json b/spec/receipt.schema.json index 17d7061..d2a9e25 100644 --- a/spec/receipt.schema.json +++ b/spec/receipt.schema.json @@ -12,7 +12,7 @@ "description": "The four terminal results. `pending` is a lifecycle state and is deliberately absent." }, "probe": {"enum": ["pass", "fail"]}, - "stateRoot": {"type": "string", "pattern": "^[0-9a-f]{32}$"} + "stateRoot": {"type": "string", "pattern": "^[0-9a-f]{64}$"} }, "properties": { "importer": {"type": "string", "minLength": 1}, diff --git a/spec/vectors/receipt-bad-state-root-invalid.json b/spec/vectors/receipt-bad-state-root-invalid.json index fa9a5cf..5812048 100644 --- a/spec/vectors/receipt-bad-state-root-invalid.json +++ b/spec/vectors/receipt-bad-state-root-invalid.json @@ -5,7 +5,7 @@ "target_root": "mem_01HX", "operation": "supersede", "pre_state_root": "not-a-digest", - "post_state_root": "0397835561846e8f0ac0c239710f752a", + "post_state_root": "0397835561846e8f0ac0c239710f752a0397835561846e8f0ac0c239710f752a", "stores": { "markdown": "verified", "vector": "verified", diff --git a/spec/vectors/receipt-binding-mutations.json b/spec/vectors/receipt-binding-mutations.json index de853d8..2379d26 100644 --- a/spec/vectors/receipt-binding-mutations.json +++ b/spec/vectors/receipt-binding-mutations.json @@ -6,8 +6,8 @@ "sequence": 2, "target_root": "mem_other", "operation": "correct", - "pre_state_root": "00000000000000000000000000000000", - "post_state_root": "11111111111111111111111111111111", + "pre_state_root": "0000000000000000000000000000000000000000000000000000000000000000", + "post_state_root": "1111111111111111111111111111111111111111111111111111111111111111", "stores": {"markdown": "unknown"}, "dispositions": {"markdown": {"fact:diet": "untouched"}}, "triad": {"negative": "fail", "positive": "pass", "preserve": "pass"}, diff --git a/spec/vectors/receipt-failed-valid.json b/spec/vectors/receipt-failed-valid.json index be4642e..56623f9 100644 --- a/spec/vectors/receipt-failed-valid.json +++ b/spec/vectors/receipt-failed-valid.json @@ -4,8 +4,8 @@ "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", - "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c", - "post_state_root": "0397835561846e8f0ac0c239710f752a", + "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c3854e8a2829938d5f9b63b1d581e5c6c", + "post_state_root": "0397835561846e8f0ac0c239710f752a0397835561846e8f0ac0c239710f752a", "stores": { "markdown": "verified", "vector": "verified", diff --git a/spec/vectors/receipt-missing-state-root-invalid.json b/spec/vectors/receipt-missing-state-root-invalid.json index 0c4cb1c..1718876 100644 --- a/spec/vectors/receipt-missing-state-root-invalid.json +++ b/spec/vectors/receipt-missing-state-root-invalid.json @@ -4,7 +4,7 @@ "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", - "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c", + "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c3854e8a2829938d5f9b63b1d581e5c6c", "stores": { "markdown": "verified", "vector": "verified", diff --git a/spec/vectors/receipt-partial-valid.json b/spec/vectors/receipt-partial-valid.json index 3115e44..107312a 100644 --- a/spec/vectors/receipt-partial-valid.json +++ b/spec/vectors/receipt-partial-valid.json @@ -4,8 +4,8 @@ "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", - "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c", - "post_state_root": "0397835561846e8f0ac0c239710f752a", + "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c3854e8a2829938d5f9b63b1d581e5c6c", + "post_state_root": "0397835561846e8f0ac0c239710f752a0397835561846e8f0ac0c239710f752a", "stores": { "markdown": "verified", "vector": "verified", diff --git a/spec/vectors/receipt-pending-as-result-invalid.json b/spec/vectors/receipt-pending-as-result-invalid.json index d0ce5d4..9cc9ff7 100644 --- a/spec/vectors/receipt-pending-as-result-invalid.json +++ b/spec/vectors/receipt-pending-as-result-invalid.json @@ -4,8 +4,8 @@ "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", - "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c", - "post_state_root": "0397835561846e8f0ac0c239710f752a", + "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c3854e8a2829938d5f9b63b1d581e5c6c", + "post_state_root": "0397835561846e8f0ac0c239710f752a0397835561846e8f0ac0c239710f752a", "stores": { "markdown": "verified", "vector": "verified", diff --git a/spec/vectors/receipt-unknown-coverage-value-invalid.json b/spec/vectors/receipt-unknown-coverage-value-invalid.json index ffa9e42..bf3f87d 100644 --- a/spec/vectors/receipt-unknown-coverage-value-invalid.json +++ b/spec/vectors/receipt-unknown-coverage-value-invalid.json @@ -4,8 +4,8 @@ "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", - "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c", - "post_state_root": "0397835561846e8f0ac0c239710f752a", + "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c3854e8a2829938d5f9b63b1d581e5c6c", + "post_state_root": "0397835561846e8f0ac0c239710f752a0397835561846e8f0ac0c239710f752a", "stores": { "markdown": "probably-fine" }, diff --git a/spec/vectors/receipt-unknown-disposition-invalid.json b/spec/vectors/receipt-unknown-disposition-invalid.json index 2976ccd..fcdadc3 100644 --- a/spec/vectors/receipt-unknown-disposition-invalid.json +++ b/spec/vectors/receipt-unknown-disposition-invalid.json @@ -4,8 +4,8 @@ "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", - "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c", - "post_state_root": "0397835561846e8f0ac0c239710f752a", + "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c3854e8a2829938d5f9b63b1d581e5c6c", + "post_state_root": "0397835561846e8f0ac0c239710f752a0397835561846e8f0ac0c239710f752a", "stores": { "markdown": "verified", "vector": "verified", diff --git a/spec/vectors/receipt-unknown-valid.json b/spec/vectors/receipt-unknown-valid.json index 371f922..b7a515d 100644 --- a/spec/vectors/receipt-unknown-valid.json +++ b/spec/vectors/receipt-unknown-valid.json @@ -4,8 +4,8 @@ "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", - "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c", - "post_state_root": "0397835561846e8f0ac0c239710f752a", + "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c3854e8a2829938d5f9b63b1d581e5c6c", + "post_state_root": "0397835561846e8f0ac0c239710f752a0397835561846e8f0ac0c239710f752a", "stores": { "prompt_cache": "unknown" }, diff --git a/spec/vectors/receipt-verified-valid.json b/spec/vectors/receipt-verified-valid.json index 647d1fe..10752f6 100644 --- a/spec/vectors/receipt-verified-valid.json +++ b/spec/vectors/receipt-verified-valid.json @@ -4,8 +4,8 @@ "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", - "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c", - "post_state_root": "0397835561846e8f0ac0c239710f752a", + "pre_state_root": "3854e8a2829938d5f9b63b1d581e5c6c3854e8a2829938d5f9b63b1d581e5c6c", + "post_state_root": "0397835561846e8f0ac0c239710f752a0397835561846e8f0ac0c239710f752a", "stores": { "markdown": "verified" }, diff --git a/tests/test_checkpoints.py b/tests/test_checkpoints.py index da6e242..cb35fa8 100644 --- a/tests/test_checkpoints.py +++ b/tests/test_checkpoints.py @@ -20,7 +20,7 @@ def checkpoint(self) -> QuarantineCheckpoint: erratum_id="err_0001", sequence=1, target_root="mem_01HX", - pre_state_root="a" * 32, + pre_state_root="a" * 64, adapters=( AdapterCheckpoint("prompt_cache", True, (), "unknown", "opaque"), AdapterCheckpoint("sqlite", True, ("fact:diet", "summary:dining"), "verified", None), @@ -55,6 +55,10 @@ def test_unsafe_erratum_id_is_rejected(self) -> None: with self.assertRaisesRegex(CheckpointError, "erratum"): replace(self.checkpoint(), erratum_id="../escape") + def test_truncated_pre_state_root_is_rejected(self) -> None: + with self.assertRaisesRegex(CheckpointError, "pre-state root"): + replace(self.checkpoint(), pre_state_root="a" * 32) + class CheckpointPersistence(unittest.TestCase): def test_atomic_round_trip_and_consumption(self) -> None: diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 704644a..bddc0b8 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -47,11 +47,11 @@ def changed_corpus(self, change) -> Path: def test_checked_in_corpus_binds_immutable_normative_sources(self) -> None: corpus = load_corpus(CORPUS, ROOT) - self.assertEqual(corpus.schema_version, 1) - self.assertEqual( - corpus.normative_target.commit, - "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + publication = json.loads( + (ROOT / "publication" / "active-surfaces.json").read_text(encoding="utf-8") ) + self.assertEqual(corpus.schema_version, 1) + self.assertEqual(corpus.normative_target.commit, publication["review_target"]["commit"]) self.assertEqual(len(corpus.cases), 5) self.assertEqual(len(corpus.validator_controls), 3) @@ -78,9 +78,12 @@ def test_normative_source_must_be_reachable_from_runtime_history(self) -> None: def test_new_current_surface_files_do_not_change_historical_manifest(self) -> None: corpus = load_corpus(CORPUS, ROOT) + publication = json.loads( + (ROOT / "publication" / "active-surfaces.json").read_text(encoding="utf-8") + ) self.assertEqual( corpus.normative_target.surface_digest, - "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12", + publication["review_target"]["surface_digest"], ) def test_quotation_drift_is_refused(self) -> None: diff --git a/tests/test_controller.py b/tests/test_controller.py index bfbd295..dbd4c17 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -9,7 +9,7 @@ import json import unittest -from prototype.adapters import Coverage +from prototype.adapters import Coverage, OpaqueAdapter from prototype.checkpoints import CheckpointError, QuarantineCheckpoint from prototype.controller import Importer, Phase from prototype.errata import Erratum, Operation, RootRegistry @@ -322,7 +322,7 @@ def test_drifted_checkpoint_is_refused_before_rebuild(self) -> None: erratum_id=checkpoint.erratum_id, sequence=checkpoint.sequence, target_root=checkpoint.target_root, - pre_state_root="b" * 32, + pre_state_root="b" * 64, adapters=checkpoint.adapters, created_at=checkpoint.created_at, ) @@ -516,6 +516,8 @@ def test_state_roots_are_deterministic(self) -> None: second = build_importer(OWNER).repair(supersede()) self.assertEqual(first.pre_state_root, second.pre_state_root) self.assertEqual(first.post_state_root, second.post_state_root) + self.assertEqual(len(first.pre_state_root), 64) + self.assertEqual(len(first.post_state_root), 64) def test_a_tampered_receipt_fails_verification(self) -> None: importer = build_importer(OWNER) @@ -565,6 +567,26 @@ def test_empty_enumeration_without_lineage_audit_cannot_verify(self) -> None: receipt.limitations, ) + def test_snapshotless_incomplete_lineage_discloses_state_root_non_binding(self) -> None: + importer = build_importer(OWNER, include_opaque=False) + adapter = SilentLineageAdapter() + adapter.snapshot = None + importer.adapters.append(adapter) + receipt = importer.repair(supersede()) + limitation = next(item for item in receipt.limitations if "silent_store" in item) + self.assertIn("state roots cannot bind", limitation) + + def test_snapshotless_opaque_store_discloses_state_root_non_binding(self) -> None: + importer = build_importer(OWNER, include_opaque=False) + opaque = OpaqueAdapter("snapshotless_opaque") + opaque.snapshot = None + importer.adapters.append(opaque) + receipt = importer.repair(supersede()) + limitation = next( + item for item in receipt.limitations if "snapshotless_opaque" in item + ) + self.assertIn("state roots cannot bind", limitation) + def test_audited_empty_scope_can_verify(self) -> None: importer = build_importer(OWNER, include_opaque=False) importer.adapters.append(AuditedEmptyAdapter()) diff --git a/tests/test_publication.py b/tests/test_publication.py index 7524560..a4846cd 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import subprocess +import sys import unittest from collections.abc import Callable from pathlib import Path @@ -19,6 +21,35 @@ def test_unmodified_publication_manifest_passes(self) -> None: result = run_checker(root, SCRIPT) self.assertEqual(result.returncode, EXIT_OK, result.stdout + result.stderr) + def test_offline_result_does_not_claim_remote_surfaces_were_verified(self) -> None: + with repo_copy() as root: + result = run_checker(root, SCRIPT) + self.assertEqual(result.returncode, EXIT_OK, result.stdout + result.stderr) + self.assertIn("offline manifest consistency", result.stdout) + self.assertIn("remote GitHub surfaces were not verified", result.stdout) + self.assertNotIn("[PASS] active surfaces", result.stdout) + + def test_remote_required_mode_is_inconclusive_without_live_evidence(self) -> None: + with repo_copy() as root: + result = subprocess.run( + [sys.executable, str(root / "scripts" / SCRIPT), "--require-remote"], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE, result.stdout) + self.assertIn("INCONCLUSIVE", result.stdout) + + def test_review_and_conformance_targets_are_identical(self) -> None: + root = Path(__file__).resolve().parents[1] + publication = json.loads( + (root / "publication" / "active-surfaces.json").read_text(encoding="utf-8") + ) + corpus = json.loads( + (root / "spec" / "adapter-conformance.json").read_text(encoding="utf-8") + ) + self.assertEqual(publication["review_target"], corpus["normative_target"]) + class PublicationGuardRejectsDrift(unittest.TestCase): def _assert_manifest_mutation_is_rejected( @@ -97,7 +128,7 @@ def test_historical_surface_cannot_be_deleted(self) -> None: def test_historical_surface_cannot_be_rewritten(self) -> None: self._assert_manifest_mutation_is_rejected( lambda payload: payload["historical_surfaces"][0].__setitem__( - "commit", "0" * 40 + "commit", "not-a-commit" ), "historical surfaces", ) diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 88ab599..e9959ee 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -37,13 +37,13 @@ SCRIPT = "check_readiness.py" -class Release040ReadinessBoundary(unittest.TestCase): +class Release041ReadinessBoundary(unittest.TestCase): def test_release_updates_version_without_upgrading_external_gates(self) -> None: root = Path(__file__).resolve().parents[1] payload = json.loads( (root / "readiness" / "production-readiness.json").read_text() ) - self.assertEqual(payload["project_version"], "0.4.0") + self.assertEqual(payload["project_version"], "0.4.1") self.assertEqual(payload["verdict"], "NOT_PROD_READY") self.assertEqual( {gate["id"]: gate["status"] for gate in payload["gates"]}, @@ -466,6 +466,25 @@ def mutate(root): result = check_after(SCRIPT, mutate) self.assert_rejected_without_traceback(result, "G2 matrix criterion") + def test_every_gate_matrix_criterion_drift_is_rejected(self) -> None: + for gate_id in ("G1", "G3", "G4", "G5"): + with self.subTest(gate_id=gate_id): + def mutate(root, selected=gate_id): + path = root / "PRODUCTION_READINESS.md" + lines = path.read_text(encoding="utf-8").splitlines() + index = next( + i for i, line in enumerate(lines) if line.startswith(f"| {selected} |") + ) + cells = lines[index][1:-1].split("|") + cells[1] = " criterion drift " + lines[index] = "|" + "|".join(cells) + "|" + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assert_rejected_without_traceback( + result, f"{gate_id} matrix criterion" + ) + def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( @@ -607,6 +626,47 @@ def mutate(payload): self.assertEqual(result.returncode, EXIT_FAIL) self.assertIn("external evidence", result.stdout) + def test_g3_g4_g5_reject_generic_or_owner_produced_external_records(self) -> None: + for gate_id in ("G3", "G4", "G5"): + with self.subTest(gate_id=gate_id): + def mutate(payload, selected=gate_id): + gate = next(g for g in payload["gates"] if g["id"] == selected) + gate["status"] = "PASS" + gate["evidence"].append( + { + "kind": "external", + "ref": "https://reviews.example.org/generic/report", + "producer": "Thomas Willner", + "observed": "2026-08-14", + } + ) + + result = self._mutated(mutate) + self.assert_rejected_without_traceback( + result, f"{gate_id} external PASS evidence" + ) + + def test_pre_corpus_review_target_fails_explicitly(self) -> None: + with repo_copy() as root: + corpus = root / "spec" / "adapter-conformance.json" + corpus_bytes = corpus.read_bytes() + corpus.unlink() + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Readiness tests"), + ("git", "add", "."), + ("git", "commit", "-q", "-m", "pre-corpus"), + ): + subprocess.run(command, cwd=root, check=True) + pre_corpus = subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + corpus.write_bytes(corpus_bytes) + with self.assertRaisesRegex(OSError, "predates the conformance corpus"): + g2_surface_digest_at_commit(pre_corpus, root) + def test_external_evidence_without_producer_is_rejected(self) -> None: def mutate(payload): gate = next(gate for gate in payload["gates"] if gate["id"] == "G2") diff --git a/tests/test_schema.py b/tests/test_schema.py index c708676..5bcbb16 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -115,6 +115,11 @@ def test_an_unknown_schema_name_raises(self) -> None: with self.assertRaises(schema.SchemaError): schema.load("nonexistent") + def test_receipt_state_roots_require_full_sha256_width(self) -> None: + receipt_schema = schema.load("receipt") + state_root = receipt_schema["$defs"]["stateRoot"] + self.assertEqual(state_root["pattern"], "^[0-9a-f]{64}$") + class ConformanceVectors(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index c84845f..7362d10 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -17,26 +17,26 @@ SCRIPT = "validate_repo.py" -class Release040Metadata(unittest.TestCase): +class Release041Metadata(unittest.TestCase): def test_version_citation_maturity_security_and_changelog_align(self) -> None: root = Path(__file__).resolve().parents[1] - self.assertEqual((root / "VERSION").read_text().strip(), "0.4.0") + self.assertEqual((root / "VERSION").read_text().strip(), "0.4.1") citation = (root / "CITATION.cff").read_text(encoding="utf-8") - self.assertIn("version: 0.4.0", citation) + self.assertIn("version: 0.4.1", citation) self.assertIn("date-released: 2026-08-14", citation) - self.assertIn("Version 0.4.0", (root / "README.md").read_text()) + self.assertIn("Version 0.4.1", (root / "README.md").read_text()) self.assertIn( "| 0.4.x | Yes |", (root / "SECURITY.md").read_text(), ) changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") - self.assertIn("## [0.4.0]", changelog) + self.assertIn("## [0.4.1]", changelog) self.assertIn("Rastislav Drahos", changelog) self.assertIn("2ba1e299b3483b9038d03387345702427608b90b", changelog) security = (root / "SECURITY.md").read_text(encoding="utf-8") - self.assertIn("with the immutable `v0.4.0` release", security) + self.assertIn("current immutable supported release is `v0.4.1`", security) self.assertIn("| 0.3.x and earlier | No |", security) - self.assertIn("first materially improved experimental release", changelog) + self.assertIn("Integrity and conformance hardening release", changelog) self.assertIn("Experimental release readiness is separate", (root / "README.md").read_text()) @@ -90,6 +90,29 @@ def test_reference_code_cannot_be_relicensed_by_specification_grant(self) -> Non "also covers `prototype/`, `scripts/`, and `tests/`", ) + def test_all_normative_markdown_is_inside_specification_materials(self) -> None: + license_text = (Path(__file__).resolve().parents[1] / "LICENSE").read_text( + encoding="utf-8" + ) + for path in ( + "HARD_PROBLEMS.md", + "PRIOR_ART.md", + "RESEARCH.md", + "PRODUCTION_READINESS.md", + "INDEPENDENT_IMPLEMENTATION.md", + "PHASE3_SYSTEMS.md", + ): + with self.subTest(path=path): + self.assertIn(path, license_text) + + def test_candidate_binding_execution_is_documented_as_trusted_code_only(self) -> None: + root = Path(__file__).resolve().parents[1] + combined = "\n".join( + (root / path).read_text(encoding="utf-8") + for path in ("INDEPENDENT_IMPLEMENTATION.md", "prototype/README.md", "SECURITY.md") + ) + self.assertIn("trusted code only", combined.casefold()) + def test_false_endorsement_protection_cannot_be_removed(self) -> None: self._assert_license_mutation_is_rejected( "does not imply endorsement, sponsorship, certification, or audit", From ae9018c5d464b8ddd0fdf2ac99577e5e7fdc562b Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 04:42:32 +0200 Subject: [PATCH 2/8] fix: align conformance surface manifests --- prototype/conformance.py | 11 ++++++----- publication/active-surfaces.json | 26 +++++++++++++++++++++----- scripts/check_publication.py | 10 +++++++--- spec/adapter-conformance.json | 4 ++-- tests/test_publication.py | 4 +--- 5 files changed, 37 insertions(+), 18 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 2f8fca5..4372ccd 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -319,7 +319,7 @@ def _git(root: Path, *args: str) -> bytes: def _surface_paths_at_commit(root: Path, commit: str) -> tuple[str, ...]: required_tests = ( "tests/test_adapters.py", "tests/test_checkpoints.py", "tests/test_cli.py", - "tests/test_controller.py", "tests/test_ed25519.py", + "tests/test_conformance.py", "tests/test_controller.py", "tests/test_ed25519.py", "tests/test_errata_feed.py", "tests/test_schema.py", "tests/test_semantic.py", "tests/test_sqlite_store.py", ) @@ -412,10 +412,11 @@ def load_corpus( if root["schema_version"] != 1: raise ConformanceInputError("corpus schema version must be 1") target = _exact(root["normative_target"], TARGET_KEYS, "normative target") - if target != _publication_target(source_root): - raise ConformanceInputError( - "normative target does not match the active publication review target" - ) + publication_target = _publication_target(source_root) + if target["commit"] != publication_target["commit"]: + raise ConformanceInputError("normative target commit is not canonical") + if target["surface_digest"] != publication_target["surface_digest"]: + raise ConformanceInputError("normative surface digest is not canonical") actual_digest = _surface_digest_at_commit(source_root, target["commit"]) if actual_digest != target["surface_digest"]: raise ConformanceInputError("normative surface digest does not match source") diff --git a/publication/active-surfaces.json b/publication/active-surfaces.json index d9d270e..9204f5c 100644 --- a/publication/active-surfaces.json +++ b/publication/active-surfaces.json @@ -1,8 +1,8 @@ { "schema_version": 3, "review_target": { - "commit": "ad36ed5e209a53aacab17751b5a183ca8a1aac1f", - "surface_digest": "3ca427bb2645517e1b1d921859a7721896644a05e32ea39fb0071a021ddc5b6d" + "commit": "cbd72633bc10720e761ad8a3dff31bb699f40dda", + "surface_digest": "c397b72b42640bd5005dd22b138d0d8761b7cd3fb94b1e5df80eb406fb5447ba" }, "release_binding": { "version": "0.4.1", @@ -40,9 +40,7 @@ ], "evidence_boundary": "recruitment-only", "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5287579823" - } - ], - "surfaces": [ + }, { "id": "g4-inspeximus-v040-target-reply", "kind": "issue-comment", @@ -58,6 +56,24 @@ "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5280210050", "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5280225709" ], + "evidence_boundary": "recruitment-only", + "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288780760" + } + ], + "surfaces": [ + { + "id": "v041-review-target", + "kind": "issue-comment", + "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288780760", + "published": "2026-08-14", + "commit": "cbd72633bc10720e761ad8a3dff31bb699f40dda", + "surface_digest": "c397b72b42640bd5005dd22b138d0d8761b7cd3fb94b1e5df80eb406fb5447ba", + "gates": ["G2", "G4"], + "roles": ["independent-reviewer"], + "mentions": [], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5287579823" + ], "evidence_boundary": "recruitment-only" } ] diff --git a/scripts/check_publication.py b/scripts/check_publication.py index e1511fb..f4a6751 100644 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -96,7 +96,7 @@ def _validate_surface( for value in values ) boundary = surface.get("evidence_boundary") - if not ( + fields_valid = ( isinstance(surface.get("id"), str) and re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", surface["id"]) is not None and surface.get("kind") in {"issue-comment", "pull-request-comment", "discussion-comment"} @@ -113,8 +113,10 @@ def _validate_surface( and len(surface["supersedes"]) == len(set(surface["supersedes"])) and all(_is_repository_url(item) for item in surface["supersedes"]) and boundary in {"recruitment-only", "publication-only"} - ): - failures.append("surface fields: invalid ID, URL, date, target, gates, supersession, or boundary") + ) + if not fields_valid: + label = "historical surfaces" if historical else "surface fields" + failures.append(f"{label}: invalid ID, URL, date, target, gates, supersession, or boundary") if not historical and ( surface.get("commit") != target["commit"] or surface.get("surface_digest") != target["surface_digest"] @@ -124,6 +126,8 @@ def _validate_surface( failures.append("evidence boundary: recruitment surfaces require roles") if boundary == "publication-only" and (surface.get("roles") or surface.get("mentions")): failures.append("evidence boundary: publication surfaces cannot recruit or mention users") + if boundary not in {"recruitment-only", "publication-only"}: + failures.append("evidence boundary: invitations and publication are not independent evidence") if historical and not _is_repository_url(surface.get("superseded_by")): failures.append("historical surfaces: superseded_by must be a repository URL") return failures diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json index aaf3918..0f4e957 100644 --- a/spec/adapter-conformance.json +++ b/spec/adapter-conformance.json @@ -3,8 +3,8 @@ "status": "candidate-internal", "evidence_boundary": "Passing this corpus is internal conformance evidence. It is not G2 or G4 evidence.", "normative_target": { - "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", - "surface_digest": "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" + "commit": "cbd72633bc10720e761ad8a3dff31bb699f40dda", + "surface_digest": "c397b72b42640bd5005dd22b138d0d8761b7cd3fb94b1e5df80eb406fb5447ba" }, "provenance": { "reported_by": "Rastislav Drahos / DanceNitra", diff --git a/tests/test_publication.py b/tests/test_publication.py index a4846cd..5f99e91 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -149,9 +149,7 @@ def mutate(payload: dict[str, object]) -> None: def test_duplicate_mention_is_rejected(self) -> None: def mutate(payload: dict[str, object]) -> None: - payload["surfaces"][0]["mentions"].append( - payload["surfaces"][0]["mentions"][0] - ) + payload["surfaces"][0]["mentions"] = ["Reviewer", "Reviewer"] self._assert_manifest_mutation_is_rejected(mutate, "unique GitHub mentions") From da088a2253533978e81672352a2a1c71606a658e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 04:44:11 +0200 Subject: [PATCH 3/8] docs: bind v0.4.1 review target --- publication/active-surfaces.json | 26 +++++++++++++++++++++----- spec/adapter-conformance.json | 4 ++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/publication/active-surfaces.json b/publication/active-surfaces.json index 9204f5c..032d3c3 100644 --- a/publication/active-surfaces.json +++ b/publication/active-surfaces.json @@ -1,8 +1,8 @@ { "schema_version": 3, "review_target": { - "commit": "cbd72633bc10720e761ad8a3dff31bb699f40dda", - "surface_digest": "c397b72b42640bd5005dd22b138d0d8761b7cd3fb94b1e5df80eb406fb5447ba" + "commit": "ae9018c5d464b8ddd0fdf2ac99577e5e7fdc562b", + "surface_digest": "fc2be2c194c801349c4e39462394b6e1e52b2e479619b68a5615e7386bfdf5b9" }, "release_binding": { "version": "0.4.1", @@ -58,9 +58,7 @@ ], "evidence_boundary": "recruitment-only", "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288780760" - } - ], - "surfaces": [ + }, { "id": "v041-review-target", "kind": "issue-comment", @@ -74,6 +72,24 @@ "supersedes": [ "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5287579823" ], + "evidence_boundary": "recruitment-only", + "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288798624" + } + ], + "surfaces": [ + { + "id": "v041-corrected-review-target", + "kind": "issue-comment", + "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288798624", + "published": "2026-08-14", + "commit": "ae9018c5d464b8ddd0fdf2ac99577e5e7fdc562b", + "surface_digest": "fc2be2c194c801349c4e39462394b6e1e52b2e479619b68a5615e7386bfdf5b9", + "gates": ["G2", "G4"], + "roles": ["independent-reviewer"], + "mentions": [], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288780760" + ], "evidence_boundary": "recruitment-only" } ] diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json index 0f4e957..5b359a5 100644 --- a/spec/adapter-conformance.json +++ b/spec/adapter-conformance.json @@ -3,8 +3,8 @@ "status": "candidate-internal", "evidence_boundary": "Passing this corpus is internal conformance evidence. It is not G2 or G4 evidence.", "normative_target": { - "commit": "cbd72633bc10720e761ad8a3dff31bb699f40dda", - "surface_digest": "c397b72b42640bd5005dd22b138d0d8761b7cd3fb94b1e5df80eb406fb5447ba" + "commit": "ae9018c5d464b8ddd0fdf2ac99577e5e7fdc562b", + "surface_digest": "fc2be2c194c801349c4e39462394b6e1e52b2e479619b68a5615e7386bfdf5b9" }, "provenance": { "reported_by": "Rastislav Drahos / DanceNitra", From 504cb659436f9d719aef27158191ced8422af022 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 05:16:24 +0200 Subject: [PATCH 4/8] fix: bind external evidence and publication history --- CHANGELOG.md | 16 +- INDEPENDENT_IMPLEMENTATION.md | 7 + PHASE3_SYSTEMS.md | 7 + README.md | 3 +- REVIEW_REQUEST.md | 11 + docs/CRYPTOGRAPHY_QUALIFICATION.md | 5 + docs/READINESS_EVIDENCE_SCHEMAS.md | 70 ++++++ prototype/conformance.py | 57 +---- prototype/surface_digest.py | 194 +++++++++++++++++ scripts/check_publication.py | 114 ++++++++++ scripts/check_readiness.py | 287 +++++++++++++++---------- tests/test_publication.py | 85 ++++++++ tests/test_readiness.py | 331 +++++++++++++++++++++++++++++ 13 files changed, 1020 insertions(+), 167 deletions(-) create mode 100644 docs/READINESS_EVIDENCE_SCHEMAS.md create mode 100644 prototype/surface_digest.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 92f135e..bbcd939 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,15 +22,25 @@ v0.4.0 published and issuing this patch release rather than withdrawing it. GitHub verification explicitly inconclusive without a freshness receipt. - Enforced ledger-to-matrix criterion equality for all six readiness gates and added gate-specific, commit-bound independent evidence contracts for G3-G5. +- Made the G3 scope parser reject malformed or unhashable tokens without a + traceback and added complete positive records for the G3-G5 contracts. +- Bound G4 validation to one shared erratum and the exact IDs and SHA-256 + digests of both adapter receipts; unrelated validator evidence cannot qualify. +- Required G5 evidence to include correction, supersession, erasure, all nine + measurements, three distinct operators, the deliberately nonconforming arm, + incomplete or opaque coverage, and mixed-artifact lineage. +- Enforced publication history against committed Git revisions so valid-looking + coordinated rewrites, deletions, and arbitrary active-surface replacement fail. - Widened receipt state roots from truncated 128-bit values to full SHA-256 and required the same width in durable quarantine checkpoints and vectors. - Disclosed state-root non-binding for every snapshot-less adapter path, including opaque and lineage-incomplete stores. -- Clarified that all six normative Markdown contracts are Specification +- Clarified every named normative Markdown contract included in Specification Materials and that conformance bindings execute trusted code only. - Replaced the silent pre-corpus digest skip with an explicit failure and bound - the adapter corpus into G2 digests while normalizing only its unavoidable - self-target pointer. + the adapter corpus into G2 digests. One shared readiness/conformance module + now normalizes only the two unavoidable self-target scalar values and keeps + every other corpus byte, including whitespace and escape spelling, bound. The production verdict remains **NOT_PROD_READY**. G2-G6 remain `BLOCKED`; this release adds no external review, independent implementation, operated-system, diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index 1a72a27..9af00c8 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -94,6 +94,13 @@ cannot also occupy the separately authored third-party validator role for its own implementation. Commercial interest and other conflicts must be disclosed; they do not erase technical evidence, but they control how it can satisfy G4. +Qualifying G4 ledger records must bind both adapter receipts to the same +erratum, immutable commit, and canonical surface digest. The validator must +name both adapter implementation IDs and exact receipt IDs and SHA-256 digests; +an unrelated validator result cannot be combined with otherwise valid adapter +records. See [`docs/READINESS_EVIDENCE_SCHEMAS.md`](docs/READINESS_EVIDENCE_SCHEMAS.md) +for the exact role-specific fields. + No per-implementer permission is required for an independently authored commercial or non-commercial implementation of the specification. The irrevocable implementation grant requires every product or service to credit diff --git a/PHASE3_SYSTEMS.md b/PHASE3_SYSTEMS.md index 19cc323..20c844f 100644 --- a/PHASE3_SYSTEMS.md +++ b/PHASE3_SYSTEMS.md @@ -20,6 +20,13 @@ Measurements include observation-to-quarantine time, known-descendant coverage, stale-behavior rate, replacement activation, collateral retention, stale-reimport resistance, opaque coverage, operator effort, and user-visible friction. +A qualifying report records all three operations and every measurement for +each system with public raw-evidence references. It also marks the intentionally +nonconforming importer, `incomplete` or `opaque` coverage system, and +mixed-artifact-lineage system explicitly. Three generic system-level pass +statements cannot satisfy G5. Exact ledger fields are defined in +[`docs/READINESS_EVIDENCE_SCHEMAS.md`](docs/READINESS_EVIDENCE_SCHEMAS.md). + ## Nomination requirements Please name: diff --git a/README.md b/README.md index bae1fcb..4e49074 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ See [PRIOR_ART.md](PRIOR_ART.md) for the feature-level comparison and [RESEARCH. | [PHASE3_SYSTEMS.md](PHASE3_SYSTEMS.md) | Nominations for future authorized three-system synthetic-data experiment. | | [docs/PUBLICATION_STRATEGY.md](docs/PUBLICATION_STRATEGY.md) | Evidence-bounded publication channels and canonical announcement copy. | | [docs/PUBLICATION_LOG.md](docs/PUBLICATION_LOG.md) | Public GitHub calls, blocked external-channel attempts, and readiness boundary. | +| [docs/READINESS_EVIDENCE_SCHEMAS.md](docs/READINESS_EVIDENCE_SCHEMAS.md) | Exact commit-bound G3-G5 external evidence records and qualification boundaries. | ## Verifying this repository @@ -185,7 +186,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.4.1 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. It hardens immutable review/release binding, publication-check claims, readiness matrix equality and external-evidence independence, full-width state roots, snapshot limitation disclosure, licence scope, and conformance-corpus digest behavior. Phase 1 and the internal Phase 2 conformance surface retain conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. G2 remains `BLOCKED`: interested-party findings and internal remediation do not replace a complete independent review of the current surface. G4 also remains `BLOCKED`: one externally authored adapter candidate exists, but two independent implementations and a separately produced third-party validator result are not established. +Version 0.4.1 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. It hardens immutable review/release binding, append-only publication history, readiness matrix equality, gate-specific external-evidence independence, full-width state roots, snapshot limitation disclosure, licence scope, and byte-preserving conformance-corpus digests. G4 validator reports must bind the same erratum and exact receipts from both adapters; G5 reports must record the complete declared three-system experiment rather than generic system passes. Phase 1 and the internal Phase 2 conformance surface retain conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. G2 remains `BLOCKED`: interested-party findings and internal remediation do not replace a complete independent review of the current surface. G4 also remains `BLOCKED`: one externally authored adapter candidate exists, but two independent implementations and a separately produced third-party validator result are not established. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md index 5802fc8..cf94590 100644 --- a/REVIEW_REQUEST.md +++ b/REVIEW_REQUEST.md @@ -103,14 +103,20 @@ byte. Canonical path enumeration is: all first-party prototype/*.py prototype/README.md spec/README.md +spec/adapter-conformance.json (only its target commit and digest string values are normalized) all first-party spec/*.schema.json all first-party spec/vectors/*.json all first-party spec/semantic/*.json ROADMAP.md THREAT_MODEL.md SECURITY.md +INDEPENDENT_IMPLEMENTATION.md +REVIEW_REQUEST.md +docs/READINESS_EVIDENCE_SCHEMAS.md tests/test_adapters.py +tests/test_checkpoints.py tests/test_cli.py +tests/test_conformance.py tests/test_controller.py tests/test_ed25519.py tests/test_errata_feed.py @@ -119,6 +125,11 @@ tests/test_semantic.py tests/test_sqlite_store.py ``` +Corpus normalization preserves every byte except the two lowercase hexadecimal +string values under the top-level `normative_target`. Whitespace, key order, +escape spelling, provenance, cases, controls, and all other bytes remain +digest-bound. Duplicate JSON keys and non-canonical target scalars fail closed. + Vendor files under `spec/vendor/` are excluded. All named groups must be nonempty. Use `python3 -c 'from scripts.check_readiness import g2_surface_digest; print(g2_surface_digest())'` from repository root to print diff --git a/docs/CRYPTOGRAPHY_QUALIFICATION.md b/docs/CRYPTOGRAPHY_QUALIFICATION.md index 8cdd638..a0d81cb 100644 --- a/docs/CRYPTOGRAPHY_QUALIFICATION.md +++ b/docs/CRYPTOGRAPHY_QUALIFICATION.md @@ -6,6 +6,11 @@ **Readiness gate:** G3 remains `BLOCKED` +Any future external qualification record must use the exact G3 scope, +implementation-build, platform, audit, and constant-time fields in +[`READINESS_EVIDENCE_SCHEMAS.md`](READINESS_EVIDENCE_SCHEMAS.md). A generic +security-review URL or malformed scope cannot satisfy the gate. + ## Decision Do not replace the reference signer yet. diff --git a/docs/READINESS_EVIDENCE_SCHEMAS.md b/docs/READINESS_EVIDENCE_SCHEMAS.md new file mode 100644 index 0000000..d276fd7 --- /dev/null +++ b/docs/READINESS_EVIDENCE_SCHEMAS.md @@ -0,0 +1,70 @@ +# External readiness evidence schemas + +This document defines the exact machine-admitted G3, G4, and G5 evidence +records. Every record is public, dated, independently produced, bound to one +immutable commit and canonical surface digest, and conflict-disclosed. A valid +`fail` report remains evidence but cannot make a gate pass. + +Shared fields are `kind=external`, a public report `ref`, non-empty `producer`, +public `producer_identity`, ISO `observed` date, exact `reviewed_commit`, exact +`surface_digest`, `conflicts` array, gate-specific `review_type`, relationship, +and independence attestation. Extra or malformed fields fail closed. + +## G3 cryptography record + +G3 uses attestation `llm-errata-independent-cryptography-review-v1`. Scope must +contain exactly: `library-build`, `constant-time`, `malformed-input-refusal`, +`key-rotation`, `key-recovery`, `revocation`, and `delegation`. + +`implementation` contains exactly `library`, `version`, `binding`, +`build_digest` (`sha256:` plus 64 lowercase hexadecimal characters), non-empty +`platforms`, `constant_time`, and `audited_build`. Qualification requires both +booleans to be true and result `pass` or `pass-with-findings`. + +## G4 adapter and validator records + +G4 uses attestation `llm-errata-independent-implementation-v1`, review type +`g4-conformance`, and one shared `erratum_id`. + +Each adapter record has `evidence_role=adapter`, relationship +`independent-implementation`, a unique `implementation_id`, and one `receipt` +containing exact `receipt_id`, `receipt_digest` (`sha256:` plus 64 lowercase +hexadecimal characters), and public `evidence_ref`. + +The separately produced validator record has `evidence_role=validator`, +relationship `independent-third-party-validator`, its own implementation and +producer identity, and `validated_receipts`. Every result names the adapter +`implementation_id`, exact receipt ID and digest, result, and public evidence +reference. G4 qualifies only when one validator reports passing results for +the exact receipts from two different adapter producers against the same +erratum, commit, and surface digest. An unrelated validator report cannot be +combined with the adapters. + +## G5 interoperability record + +G5 uses attestation `llm-errata-independent-interoperability-review-v1`, review +type `phase3-interoperability`, relationship `independent-experiment-report`, +`synthetic_data=true`, `user_controlled_root=true`, and one non-empty `root_id`. + +Exactly three independently operated systems are required, with different +operator identities. Every system names version and evidence, completes +`correction`, `supersession`, and `erasure`, and records all nine metrics: + +- observation-to-quarantine time; +- known-descendant coverage; +- stale-behavior rate; +- replacement activation; +- collateral retention; +- stale-reimport resistance; +- opaque coverage; +- operator effort; and +- user-visible friction. + +Every operation and measurement has its own public evidence reference. The +three-system set must include at least one intentionally nonconforming importer, +one `incomplete` or `opaque` coverage system, and one mixed-artifact-lineage +system. Three generic passing system summaries are insufficient. + +These schemas establish structural admission only. Maintainer review still +must verify real identities, authorship, conflicts, authorization, raw evidence, +and whether the claimed measurements support the report. diff --git a/prototype/conformance.py b/prototype/conformance.py index 4372ccd..b8fe7b0 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -18,6 +18,11 @@ from pathlib import Path from typing import Any, Callable +from prototype.surface_digest import ( + SurfaceDigestError, + g2_surface_digest_at_commit, +) + class ConformanceInputError(ValueError): """Corpus or source evidence cannot support a conformance run.""" @@ -316,55 +321,11 @@ def _git(root: Path, *args: str) -> bytes: return result.stdout -def _surface_paths_at_commit(root: Path, commit: str) -> tuple[str, ...]: - required_tests = ( - "tests/test_adapters.py", "tests/test_checkpoints.py", "tests/test_cli.py", - "tests/test_conformance.py", "tests/test_controller.py", "tests/test_ed25519.py", - "tests/test_errata_feed.py", "tests/test_schema.py", - "tests/test_semantic.py", "tests/test_sqlite_store.py", - ) - listed = _git(root, "ls-tree", "-r", "--name-only", commit).decode("utf-8").splitlines() - files = set(listed) - groups = ( - tuple(sorted(path for path in files if re.fullmatch(r"prototype/[^/]+\.py", path))), - ("prototype/README.md", "spec/README.md"), - ("spec/adapter-conformance.json",), - tuple(sorted(path for path in files if re.fullmatch(r"spec/[^/]+\.schema\.json", path))), - tuple(sorted(path for path in files if re.fullmatch(r"spec/vectors/[^/]+\.json", path))), - tuple(sorted(path for path in files if re.fullmatch(r"spec/semantic/[^/]+\.json", path))), - ("ROADMAP.md", "THREAT_MODEL.md", "SECURITY.md"), - required_tests, - ) - paths = tuple(sorted(item for group in groups for item in group)) - if any(path not in files for path in paths): - raise ConformanceInputError("canonical surface is incomplete") - return paths - - def _surface_digest_at_commit(root: Path, commit: str) -> str: - digest = hashlib.sha256() - for relative in _surface_paths_at_commit(root, commit): - content = _git(root, "show", f"{commit}:{relative}") - if relative == "spec/adapter-conformance.json": - try: - payload = json.loads(content.decode("utf-8")) - target = payload["normative_target"] - if not isinstance(target, dict): - raise TypeError - target["commit"] = "0" * 40 - target["surface_digest"] = "0" * 64 - content = json.dumps( - payload, sort_keys=True, separators=(",", ":") - ).encode("utf-8") - except (UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError) as error: - raise ConformanceInputError( - "conformance corpus target metadata is malformed" - ) from error - digest.update(relative.encode("utf-8")) - digest.update(b"\0") - digest.update(content) - digest.update(b"\0") - return digest.hexdigest() + try: + return g2_surface_digest_at_commit(root, commit) + except SurfaceDigestError as error: + raise ConformanceInputError(str(error)) from error def _validate_outcome(value: object, operation: str, label: str) -> dict[str, Any]: diff --git a/prototype/surface_digest.py b/prototype/surface_digest.py new file mode 100644 index 0000000..e9b3882 --- /dev/null +++ b/prototype/surface_digest.py @@ -0,0 +1,194 @@ +"""Canonical Phase 2 surface manifest and byte-preserving digest helpers.""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from pathlib import Path + + +REQUIRED_TESTS = ( + "tests/test_adapters.py", + "tests/test_checkpoints.py", + "tests/test_cli.py", + "tests/test_conformance.py", + "tests/test_controller.py", + "tests/test_ed25519.py", + "tests/test_errata_feed.py", + "tests/test_schema.py", + "tests/test_semantic.py", + "tests/test_sqlite_store.py", +) +REQUIRED_EVIDENCE_CONTRACT = "docs/READINESS_EVIDENCE_SCHEMAS.md" +CORPUS_PATH = "spec/adapter-conformance.json" +GIT_TIMEOUT_SECONDS = 10.0 + + +class SurfaceDigestError(OSError): + """Source bytes cannot support a canonical surface digest.""" + + +def surface_digest_from_bytes(entries: list[tuple[str, bytes]]) -> str: + """Hash ordered path and content pairs without changing their bytes.""" + + digest = hashlib.sha256() + for relative, content in entries: + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(content) + digest.update(b"\0") + return digest.hexdigest() + + +def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _object_end(text: str, start: int) -> int: + depth = 0 + in_string = False + escaped = False + for index in range(start, len(text)): + token = text[index] + if in_string: + if escaped: + escaped = False + elif token == "\\": + escaped = True + elif token == '"': + in_string = False + continue + if token == '"': + in_string = True + elif token == "{": + depth += 1 + elif token == "}": + depth -= 1 + if depth == 0: + return index + raise SurfaceDigestError("conformance corpus target metadata is malformed") + + +def normalize_corpus_target(content: bytes) -> bytes: + """Zero only target scalar values while preserving every other UTF-8 byte.""" + + try: + text = content.decode("utf-8") + payload = json.loads(text, object_pairs_hook=_reject_duplicate_keys) + target = payload["normative_target"] + if not isinstance(payload, dict) or not isinstance(target, dict): + raise TypeError + if set(target) != {"commit", "surface_digest"}: + raise TypeError + if re.fullmatch(r"[0-9a-f]{40}", target["commit"]) is None: + raise TypeError + if re.fullmatch(r"[0-9a-f]{64}", target["surface_digest"]) is None: + raise TypeError + except (UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError, ValueError) as error: + raise SurfaceDigestError("conformance corpus target metadata is malformed") from error + + target_markers = list(re.finditer(r'"normative_target"\s*:\s*\{', text)) + if len(target_markers) != 1: + raise SurfaceDigestError("conformance corpus target metadata is malformed") + start = target_markers[0].end() - 1 + end = _object_end(text, start) + block = text[start : end + 1] + replacements: list[tuple[int, int, str]] = [] + for key, width in (("commit", 40), ("surface_digest", 64)): + matches = list(re.finditer( + rf'"{key}"\s*:\s*"([0-9a-f]{{{width}}})"', block + )) + if len(matches) != 1: + raise SurfaceDigestError("conformance corpus target metadata is malformed") + replacements.append(( + start + matches[0].start(1), + start + matches[0].end(1), + "0" * width, + )) + for value_start, value_end, replacement in sorted(replacements, reverse=True): + text = text[:value_start] + replacement + text[value_end:] + return text.encode("utf-8") + + +def review_surface_content(relative: str, content: bytes) -> bytes: + """Normalize the canonical corpus target and leave every other file verbatim.""" + + return normalize_corpus_target(content) if relative == CORPUS_PATH else content + + +def _paths_from_files(files: set[str]) -> tuple[str, ...]: + groups = ( + tuple(sorted(path for path in files if re.fullmatch(r"prototype/[^/]+\.py", path))), + ("prototype/README.md", "spec/README.md"), + (CORPUS_PATH,), + tuple(sorted(path for path in files if re.fullmatch(r"spec/[^/]+\.schema\.json", path))), + tuple(sorted(path for path in files if re.fullmatch(r"spec/vectors/[^/]+\.json", path))), + tuple(sorted(path for path in files if re.fullmatch(r"spec/semantic/[^/]+\.json", path))), + ( + "ROADMAP.md", "THREAT_MODEL.md", "SECURITY.md", + "INDEPENDENT_IMPLEMENTATION.md", "REVIEW_REQUEST.md", + REQUIRED_EVIDENCE_CONTRACT, + ), + REQUIRED_TESTS, + ) + if any(not group for group in groups): + raise SurfaceDigestError("canonical G2 surface is incomplete") + paths = tuple(sorted(item for group in groups for item in group)) + if CORPUS_PATH not in files: + raise SurfaceDigestError("reviewed commit predates the conformance corpus") + if any(path not in files for path in paths): + raise SurfaceDigestError("canonical G2 surface is incomplete") + return paths + + +def g2_surface_files(root: Path) -> tuple[str, ...]: + files = { + path.relative_to(root).as_posix() + for path in root.rglob("*") + if path.is_file() and ".git" not in path.relative_to(root).parts + } + return _paths_from_files(files) + + +def _git(root: Path, *args: str) -> bytes: + try: + result = subprocess.run( + ["git", *args], cwd=root, capture_output=True, check=False, + timeout=GIT_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as error: + raise SurfaceDigestError("Git source verification timed out") from error + if result.returncode != 0: + raise SurfaceDigestError("reviewed commit is unavailable") + return result.stdout + + +def g2_surface_files_at_commit(root: Path, commit: str) -> tuple[str, ...]: + files = set( + _git(root, "ls-tree", "-r", "--name-only", commit).decode("utf-8").splitlines() + ) + return _paths_from_files(files) + + +def g2_surface_digest(root: Path) -> str: + return surface_digest_from_bytes([ + (relative, review_surface_content(relative, (root / relative).read_bytes())) + for relative in g2_surface_files(root) + ]) + + +def g2_surface_digest_at_commit(root: Path, commit: str) -> str: + return surface_digest_from_bytes([ + ( + relative, + review_surface_content(relative, _git(root, "show", f"{commit}:{relative}")), + ) + for relative in g2_surface_files_at_commit(root, commit) + ]) diff --git a/scripts/check_publication.py b/scripts/check_publication.py index f4a6751..ae6ecda 100644 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -139,6 +139,119 @@ def _git(*args: str) -> subprocess.CompletedProcess[str]: ) +def _manifest_history() -> tuple[list[dict[str, object]], list[str]]: + """Read every committed manifest revision from oldest to newest.""" + + failures: list[str] = [] + log = _git("log", "--format=%H", "--", "publication/active-surfaces.json") + if log.returncode != 0: + return [], ["publication history: committed manifest history is unavailable"] + manifests: list[dict[str, object]] = [] + for commit in reversed(log.stdout.splitlines()): + shown = _git("show", f"{commit}:publication/active-surfaces.json") + try: + payload = json.loads(shown.stdout) + except (json.JSONDecodeError, TypeError): + failures.append( + f"publication history: manifest at {commit[:12]} is unreadable" + ) + continue + if not ( + shown.returncode == 0 + and isinstance(payload, dict) + and isinstance(payload.get("surfaces"), list) + ): + failures.append( + f"publication history: manifest at {commit[:12]} lacks surface records" + ) + continue + if payload.get("schema_version") == 1: + continue + if not isinstance(payload.get("historical_surfaces"), list): + failures.append( + f"publication history: manifest at {commit[:12]} lacks historical records" + ) + continue + manifests.append(payload) + return manifests, failures + + +def _same_active_record(left: object, right: object) -> bool: + return ( + isinstance(left, dict) + and isinstance(right, dict) + and set(left) == SURFACE_KEYS + and set(right) == SURFACE_KEYS + and left == right + ) + + +def _historical_version_of(active: object, historical: object) -> bool: + return ( + isinstance(active, dict) + and isinstance(historical, dict) + and set(active) == SURFACE_KEYS + and set(historical) == HISTORICAL_SURFACE_KEYS + and all(historical.get(key) == active.get(key) for key in SURFACE_KEYS) + and _is_repository_url(historical.get("superseded_by")) + ) + + +def _validate_append_only_history(payload: dict[str, object]) -> list[str]: + """Require every committed surface transition to preserve exact prior records.""" + + manifests, failures = _manifest_history() + if failures: + return failures + if not manifests: + return [] + if manifests[-1] != payload: + manifests.append(payload) + + history_failed = False + mapping_failed = False + for previous, current in zip(manifests, manifests[1:]): + prior_history = previous["historical_surfaces"] + current_history = current["historical_surfaces"] + prior_active = previous["surfaces"] + current_active = current["surfaces"] + + if any(record not in current_history for record in prior_history): + history_failed = True + + removed_active = [] + for record in prior_active: + if any(_same_active_record(record, candidate) for candidate in current_active): + continue + if not any( + _historical_version_of(record, candidate) + for candidate in current_history + ): + history_failed = True + removed_active.append(record) + + removed_urls = { + record.get("url") for record in removed_active if isinstance(record, dict) + } + if removed_urls: + for record in current_active: + if any(_same_active_record(record, candidate) for candidate in prior_active): + continue + supersedes = record.get("supersedes") if isinstance(record, dict) else None + if not isinstance(supersedes, list) or not removed_urls.intersection(supersedes): + mapping_failed = True + + if history_failed: + failures.append( + "historical surfaces: committed records must remain exact and append-only" + ) + if mapping_failed: + failures.append( + "active surfaces: replacement mapping must supersede the prior active URL" + ) + return failures + + def validate_manifest(payload: object) -> list[str]: failures: list[str] = [] if not isinstance(payload, dict) or set(payload) != ROOT_KEYS: @@ -227,6 +340,7 @@ def validate_manifest(payload: object) -> list[str]: failures.append("unique GitHub mentions: each identity may be notified only once") if (ROOT / ".git").exists() and _valid_target(target): + failures.extend(_validate_append_only_history(payload)) commit = target["commit"] ancestor = _git("merge-base", "--is-ancestor", commit, "HEAD") if ancestor.returncode != 0: diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 9d92fc2..7b2ed7e 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -5,16 +5,27 @@ import json import re -import hashlib import math import operator import subprocess import sys from datetime import date, datetime +from itertools import combinations from pathlib import Path from urllib.parse import urlparse ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from prototype.surface_digest import ( + g2_surface_digest as _shared_g2_surface_digest, + g2_surface_digest_at_commit as _shared_g2_surface_digest_at_commit, + g2_surface_files as _shared_g2_surface_files, + review_surface_content, + surface_digest_from_bytes, +) + LEDGER = ROOT / "readiness" / "production-readiness.json" MATRIX = ROOT / "PRODUCTION_READINESS.md" REQUIRED_GATES = {"G1", "G2", "G3", "G4", "G5", "G6"} @@ -39,18 +50,6 @@ ) URN_RE = re.compile(r"^urn:[A-Za-z0-9][A-Za-z0-9-]{1,31}:[^\s]+$") G2_ATTESTATION = "llm-errata-independent-review-v1" -G2_REQUIRED_TESTS = ( - "tests/test_adapters.py", - "tests/test_checkpoints.py", - "tests/test_cli.py", - "tests/test_conformance.py", - "tests/test_controller.py", - "tests/test_ed25519.py", - "tests/test_errata_feed.py", - "tests/test_schema.py", - "tests/test_semantic.py", - "tests/test_sqlite_store.py", -) G2_SCOPE = frozenset( { "schemas", @@ -65,12 +64,46 @@ ) G2_RESULTS = {"pass", "pass-with-findings", "fail"} G3_ATTESTATION = "llm-errata-independent-cryptography-review-v1" +G3_ENTRY_KEYS = { + "kind", "ref", "producer", "producer_identity", "observed", "review_type", + "reviewed_commit", "scope", "result", "relationship", "conflicts", + "independence_attestation", "surface_digest", "implementation", +} G3_SCOPE = frozenset({ "library-build", "constant-time", "malformed-input-refusal", "key-rotation", "key-recovery", "revocation", "delegation", }) G4_ATTESTATION = "llm-errata-independent-implementation-v1" +G4_COMMON_KEYS = { + "kind", "ref", "producer", "producer_identity", "observed", "review_type", + "reviewed_commit", "surface_digest", "result", "relationship", "conflicts", + "independence_attestation", "evidence_role", "implementation_id", "erratum_id", +} +G4_RECEIPT_KEYS = {"receipt_id", "receipt_digest", "evidence_ref"} +G4_VALIDATION_KEYS = { + "implementation_id", "receipt_id", "receipt_digest", "result", "evidence_ref", +} G5_ATTESTATION = "llm-errata-independent-interoperability-review-v1" +G5_ENTRY_KEYS = { + "kind", "ref", "producer", "producer_identity", "observed", "review_type", + "reviewed_commit", "surface_digest", "result", "relationship", "conflicts", + "independence_attestation", "synthetic_data", "user_controlled_root", + "root_id", "systems", +} +G5_SYSTEM_KEYS = { + "name", "version", "operator", "operator_identity", "evidence_ref", "result", + "independently_operated", "intentionally_nonconforming", "coverage", + "mixed_artifact_lineage", "operations", "measurements", +} +G5_OPERATION_KEYS = {"operation", "completed", "evidence_ref"} +G5_OPERATIONS = frozenset({"correction", "supersession", "erasure"}) +G5_MEASUREMENT_KEYS = {"metric", "value", "unit", "evidence_ref"} +G5_MEASUREMENTS = frozenset({ + "observation-to-quarantine-time", "known-descendant-coverage", + "stale-behavior-rate", "replacement-activation", "collateral-retention", + "stale-reimport-resistance", "opaque-coverage", "operator-effort", + "user-visible-friction", +}) G6_ATTESTATION = "llm-errata-independent-operational-review-v1" G6_SCOPE = frozenset( { @@ -189,94 +222,19 @@ def valid_g2_identity_ref(value: object) -> bool: def g2_surface_files(root: Path = ROOT) -> tuple[str, ...]: - """Return comprehensive sorted first-party Phase 2 review manifest.""" - - groups = ( - tuple(sorted((root / "prototype").glob("*.py"))), - (root / "prototype" / "README.md",), - (root / "spec" / "README.md",), - (root / "spec" / "adapter-conformance.json",), - tuple(sorted((root / "spec").glob("*.schema.json"))), - tuple(sorted((root / "spec" / "vectors").glob("*.json"))), - tuple(sorted((root / "spec" / "semantic").glob("*.json"))), - tuple(root / path for path in ("ROADMAP.md", "THREAT_MODEL.md", "SECURITY.md")), - tuple(root / path for path in G2_REQUIRED_TESTS), - ) - if any(not group for group in groups) or any(not path.is_file() for group in groups for path in group): - raise OSError("canonical G2 surface is incomplete") - return tuple( - sorted(path.relative_to(root).as_posix() for group in groups for path in group) - ) - - -def surface_digest_from_bytes(entries: list[tuple[str, bytes]]) -> str: - """SHA-256 over `relative path + NUL + raw bytes + NUL` ordered entries.""" - - digest = hashlib.sha256() - for relative, content in entries: - digest.update(relative.encode("utf-8")) - digest.update(b"\0") - digest.update(content) - digest.update(b"\0") - return digest.hexdigest() - - -def review_surface_content(relative: str, content: bytes) -> bytes: - """Normalize only the corpus's self-referential target pointer. + """Return the comprehensive sorted first-party Phase 2 review manifest.""" - Cases, provenance, controls, and every other corpus byte remain digest-bound. - The target commit and digest are validated separately. Normalizing those two - fields lets a later packaging commit point at the immutable source commit - without requiring a Git commit to contain its own hash. - """ - - if relative != "spec/adapter-conformance.json": - return content - try: - payload = json.loads(content.decode("utf-8")) - target = payload["normative_target"] - if not isinstance(target, dict): - raise TypeError - target["commit"] = "0" * 40 - target["surface_digest"] = "0" * 64 - except (UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError) as error: - raise OSError("conformance corpus target metadata is malformed") from error - return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return _shared_g2_surface_files(root) def g2_surface_digest(root: Path = ROOT) -> str: - return surface_digest_from_bytes( - [ - (relative, review_surface_content(relative, (root / relative).read_bytes())) - for relative in g2_surface_files(root) - ] - ) + return _shared_g2_surface_digest(root) def g2_surface_digest_at_commit(commit: str, root: Path = ROOT) -> str: if not reviewed_commit_exists(commit, root): raise OSError("reviewed commit is unavailable") - listed = subprocess.run( - ["git", "ls-tree", "-r", "--name-only", commit], - cwd=root, - capture_output=True, - check=False, - text=True, - ) - if listed.returncode != 0: - raise OSError("reviewed commit tree is unavailable") - commit_files = set(listed.stdout.splitlines()) - entries = [] - for relative in g2_surface_files(root): - if relative == "spec/adapter-conformance.json" and relative not in commit_files: - raise OSError("reviewed commit predates the conformance corpus") - result = subprocess.run( - ["git", "show", f"{commit}:{relative}"], cwd=root, capture_output=True, check=False - ) - if result.returncode != 0: - raise OSError(f"reviewed commit lacks {relative}") - entries.append((relative, review_surface_content(relative, result.stdout))) - return surface_digest_from_bytes(entries) + return _shared_g2_surface_digest_at_commit(root, commit) def reviewed_commit_exists(value: str, root: Path = ROOT) -> bool: @@ -385,11 +343,12 @@ def _surface_digest_for_files_at_commit( G3_SURFACE_FILES = ( "prototype/ed25519.py", "prototype/errata.py", "prototype/signing.py", "THREAT_MODEL.md", "docs/CRYPTOGRAPHY_QUALIFICATION.md", + "docs/READINESS_EVIDENCE_SCHEMAS.md", "tests/test_ed25519.py", "tests/test_errata_feed.py", ) G5_SURFACE_FILES = ( "PHASE3_SYSTEMS.md", "ROADMAP.md", "spec/erratum.schema.json", - "spec/receipt.schema.json", + "spec/receipt.schema.json", "docs/READINESS_EVIDENCE_SCHEMAS.md", ) @@ -440,13 +399,17 @@ def qualifying_g3_security_evidence( ) or not isinstance(entry, dict): return False implementation = entry.get("implementation") + scope = entry.get("scope") return ( - entry.get("review_type") == "production-cryptography" + set(entry) == G3_ENTRY_KEYS + and entry.get("review_type") == "production-cryptography" and entry.get("relationship") == "independent-third-party" and entry.get("result") in {"pass", "pass-with-findings"} - and isinstance(entry.get("scope"), list) - and set(entry["scope"]) == G3_SCOPE - and len(entry["scope"]) == len(G3_SCOPE) + and isinstance(scope, list) + and all(isinstance(token, str) for token in scope) + and len(scope) == len(G3_SCOPE) + and len(scope) == len(set(scope)) + and set(scope) == G3_SCOPE and _exact_dict(implementation, { "library", "version", "binding", "build_digest", "platforms", "constant_time", "audited_build", @@ -474,13 +437,48 @@ def valid_g4_implementation_evidence( "independent-implementation" if role == "adapter" else "independent-third-party-validator" ) - return ( + if not ( entry.get("review_type") == "g4-conformance" and role in {"adapter", "validator"} and entry.get("relationship") == expected_relationship and entry.get("result") in {"pass", "pass-with-findings"} and _nonempty(entry.get("implementation_id")) - ) + and _nonempty(entry.get("erratum_id")) + ): + return False + if role == "adapter": + receipt = entry.get("receipt") + return ( + set(entry) == G4_COMMON_KEYS | {"receipt"} + and _exact_dict(receipt, G4_RECEIPT_KEYS) + and _nonempty(receipt["receipt_id"]) + and isinstance(receipt["receipt_digest"], str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", receipt["receipt_digest"]) + is not None + and valid_g2_report_ref(receipt["evidence_ref"]) + ) + validated = entry.get("validated_receipts") + if not ( + set(entry) == G4_COMMON_KEYS | {"validated_receipts"} + and isinstance(validated, list) + and len(validated) >= 2 + ): + return False + for result in validated: + if not ( + _exact_dict(result, G4_VALIDATION_KEYS) + and _nonempty(result["implementation_id"]) + and _nonempty(result["receipt_id"]) + and isinstance(result["receipt_digest"], str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", result["receipt_digest"]) + is not None + and result["result"] in {"pass", "pass-with-findings"} + and valid_g2_report_ref(result["evidence_ref"]) + ): + return False + identities = [result["implementation_id"] for result in validated] + receipts = [result["receipt_id"] for result in validated] + return len(identities) == len(set(identities)) and len(receipts) == len(set(receipts)) def qualifying_g4_evidence( @@ -494,16 +492,44 @@ def qualifying_g4_evidence( validators = [entry for entry in valid if entry["evidence_role"] == "validator"] if len(adapters) < 2 or not validators: return False - adapter_identities = {entry["producer_identity"] for entry in adapters} - adapter_implementations = {entry["implementation_id"] for entry in adapters} - validator_identities = {entry["producer_identity"] for entry in validators} - targets = {(entry["reviewed_commit"], entry["surface_digest"]) for entry in valid} - return ( - len(adapter_identities) >= 2 - and len(adapter_implementations) >= 2 - and validator_identities.isdisjoint(adapter_identities) - and len(targets) == 1 - ) + all_adapter_identities = {entry["producer_identity"] for entry in adapters} + for validator in validators: + if validator["producer_identity"] in all_adapter_identities: + continue + validated = { + ( + result["implementation_id"], result["receipt_id"], + result["receipt_digest"], + ) + for result in validator["validated_receipts"] + if result["result"] in {"pass", "pass-with-findings"} + } + validator_target = (validator["reviewed_commit"], validator["surface_digest"]) + for first, second in combinations(adapters, 2): + if first["producer_identity"] == second["producer_identity"]: + continue + if first["implementation_id"] == second["implementation_id"]: + continue + if not ( + first["erratum_id"] == second["erratum_id"] == validator["erratum_id"] + ): + continue + if any( + (adapter["reviewed_commit"], adapter["surface_digest"]) + != validator_target + for adapter in (first, second) + ): + continue + expected = { + ( + adapter["implementation_id"], adapter["receipt"]["receipt_id"], + adapter["receipt"]["receipt_digest"], + ) + for adapter in (first, second) + } + if expected.issubset(validated): + return True + return False def qualifying_g5_interoperability_evidence( @@ -516,28 +542,59 @@ def qualifying_g5_interoperability_evidence( return False systems = entry.get("systems") if not ( - entry.get("review_type") == "phase3-interoperability" + set(entry) == G5_ENTRY_KEYS + and entry.get("review_type") == "phase3-interoperability" and entry.get("relationship") == "independent-experiment-report" and entry.get("result") in {"pass", "pass-with-findings"} and entry.get("synthetic_data") is True + and entry.get("user_controlled_root") is True and _nonempty(entry.get("root_id")) and isinstance(systems, list) and len(systems) == 3 ): return False for system in systems: if not ( - _exact_dict(system, { - "name", "version", "operator", "operator_identity", - "evidence_ref", "result", "independently_operated", - }) + _exact_dict(system, G5_SYSTEM_KEYS) and all(_nonempty(system[key]) for key in ("name", "version", "operator")) and valid_g2_identity_ref(system["operator_identity"]) and valid_g2_report_ref(system["evidence_ref"]) and system["result"] == "pass" and system["independently_operated"] is True + and isinstance(system["intentionally_nonconforming"], bool) + and system["coverage"] in {"complete", "incomplete", "opaque"} + and isinstance(system["mixed_artifact_lineage"], bool) ): return False - return len({system["operator_identity"] for system in systems}) == 3 + operations = system["operations"] + if not ( + isinstance(operations, list) + and len(operations) == len(G5_OPERATIONS) + and all(_exact_dict(operation, G5_OPERATION_KEYS) for operation in operations) + and all(isinstance(operation["operation"], str) for operation in operations) + and {operation["operation"] for operation in operations} == G5_OPERATIONS + and all(operation["completed"] is True for operation in operations) + and all(valid_g2_report_ref(operation["evidence_ref"]) for operation in operations) + ): + return False + measurements = system["measurements"] + if not ( + isinstance(measurements, list) + and len(measurements) == len(G5_MEASUREMENTS) + and all(_exact_dict(measurement, G5_MEASUREMENT_KEYS) for measurement in measurements) + and all(isinstance(measurement["metric"], str) for measurement in measurements) + and {measurement["metric"] for measurement in measurements} == G5_MEASUREMENTS + and all(_finite_number(measurement["value"]) for measurement in measurements) + and all(_nonempty(measurement["unit"]) for measurement in measurements) + and all(valid_g2_report_ref(measurement["evidence_ref"]) for measurement in measurements) + ): + return False + return ( + len({system["name"] for system in systems}) == 3 + and len({system["operator_identity"] for system in systems}) == 3 + and any(system["intentionally_nonconforming"] for system in systems) + and any(system["coverage"] in {"incomplete", "opaque"} for system in systems) + and any(system["mixed_artifact_lineage"] for system in systems) + ) def g6_surface_files(root: Path = ROOT) -> tuple[str, ...]: diff --git a/tests/test_publication.py b/tests/test_publication.py index 5f99e91..db6a5cd 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -7,14 +7,62 @@ import sys import unittest from collections.abc import Callable +from contextlib import contextmanager from pathlib import Path +from scripts.check_readiness import g2_surface_digest_at_commit from tests.support import EXIT_FAIL, EXIT_INCONCLUSIVE, EXIT_OK, check_after, repo_copy, run_checker SCRIPT = "check_publication.py" +@contextmanager +def bound_publication_repo(): + with repo_copy() as root: + publication_path = root / "publication" / "active-surfaces.json" + corpus_path = root / "spec" / "adapter-conformance.json" + publication = json.loads(publication_path.read_text(encoding="utf-8")) + corpus = json.loads(corpus_path.read_text(encoding="utf-8")) + placeholder = {"commit": "0" * 40, "surface_digest": "0" * 64} + corpus["normative_target"] = placeholder + publication_path.unlink() + corpus_path.write_text(json.dumps(corpus, indent=2) + "\n", encoding="utf-8") + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Publication tests"), + ("git", "config", "gc.auto", "0"), + ("git", "add", "."), + ("git", "commit", "-q", "-m", "publication source"), + ): + subprocess.run(command, cwd=root, check=True) + source_commit = subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + target = { + "commit": source_commit, + "surface_digest": g2_surface_digest_at_commit(source_commit, root), + } + publication["review_target"] = target + for surface in publication["surfaces"]: + surface["commit"] = target["commit"] + surface["surface_digest"] = target["surface_digest"] + corpus["normative_target"] = target + publication_path.write_text(json.dumps(publication, indent=2) + "\n", encoding="utf-8") + corpus_path.write_text(json.dumps(corpus, indent=2) + "\n", encoding="utf-8") + subprocess.run( + ("git", "add", "publication/active-surfaces.json", "spec/adapter-conformance.json"), + cwd=root, check=True, + ) + subprocess.run( + ("git", "commit", "-q", "-m", "bind publication target"), + cwd=root, check=True, + ) + yield root + + class PublicationGuardPasses(unittest.TestCase): def test_unmodified_publication_manifest_passes(self) -> None: with repo_copy() as root: @@ -65,6 +113,18 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout + result.stderr) self.assertIn(diagnostic, result.stdout) + def _assert_history_mutation_is_rejected( + self, mutate_payload: Callable[[dict[str, object]], None], diagnostic: str + ) -> None: + with bound_publication_repo() as root: + path = root / "publication" / "active-surfaces.json" + payload = json.loads(path.read_text(encoding="utf-8")) + mutate_payload(payload) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + result = run_checker(root, SCRIPT) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout + result.stderr) + self.assertIn(diagnostic, result.stdout) + def test_stale_commit_is_rejected(self) -> None: self._assert_manifest_mutation_is_rejected( lambda payload: payload["review_target"].__setitem__( @@ -133,6 +193,31 @@ def test_historical_surface_cannot_be_rewritten(self) -> None: "historical surfaces", ) + def test_one_of_multiple_historical_surfaces_cannot_be_deleted(self) -> None: + self._assert_history_mutation_is_rejected( + lambda payload: payload["historical_surfaces"].pop(1), + "historical surfaces", + ) + + def test_well_formed_historical_surface_rewrite_is_rejected(self) -> None: + def mutate(payload: dict[str, object]) -> None: + payload["historical_surfaces"][0]["commit"] = "a" * 40 + payload["historical_surfaces"][0]["url"] = ( + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-9999999999" + ) + + self._assert_history_mutation_is_rejected(mutate, "historical surfaces") + + def test_well_formed_active_surface_replacement_is_rejected(self) -> None: + def mutate(payload: dict[str, object]) -> None: + surface = payload["surfaces"][0] + surface["id"] = "arbitrary-current-surface" + surface["url"] = ( + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-9999999998" + ) + + self._assert_history_mutation_is_rejected(mutate, "active surfaces") + def test_duplicate_surface_url_is_rejected(self) -> None: def mutate(payload: dict[str, object]) -> None: payload["surfaces"].append(dict(payload["surfaces"][0])) diff --git a/tests/test_readiness.py b/tests/test_readiness.py index e9959ee..5cd91ef 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -8,21 +8,31 @@ import subprocess import tempfile import unittest +from contextlib import contextmanager from pathlib import Path +from prototype.conformance import _surface_digest_at_commit as conformance_surface_digest_at_commit from scripts.check_readiness import ( G2_MATRIX_CURRENT_EVIDENCE, + G3_SCOPE, + G3_SURFACE_FILES, + G5_SURFACE_FILES, G6_SCOPE, g2_surface_digest, g2_surface_digest_at_commit, g2_surface_files, g6_surface_digest, + qualifying_g3_security_evidence, + qualifying_g4_evidence, + qualifying_g5_interoperability_evidence, qualifying_g6_operational_evidence, qualifying_g2_review_evidence, valid_external_evidence, valid_g2_review_evidence, + valid_g4_implementation_evidence, valid_g6_operational_evidence, markdown_value, + surface_digest_from_bytes, ) from tests.support import ( EXIT_FAIL, @@ -37,6 +47,31 @@ SCRIPT = "check_readiness.py" +@contextmanager +def committed_repo(): + with repo_copy() as root: + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Readiness tests"), + ("git", "config", "gc.auto", "0"), + ("git", "add", "."), + ("git", "commit", "-q", "-m", "gate evidence baseline"), + ): + subprocess.run(command, cwd=root, check=True) + commit = subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + yield root, commit + + +def gate_surface_digest(root: Path, files: tuple[str, ...]) -> str: + return surface_digest_from_bytes( + [(relative, (root / relative).read_bytes()) for relative in sorted(files)] + ) + + class Release041ReadinessBoundary(unittest.TestCase): def test_release_updates_version_without_upgrading_external_gates(self) -> None: root = Path(__file__).resolve().parents[1] @@ -64,6 +99,301 @@ def test_release_updates_version_without_upgrading_external_gates(self) -> None: class ReadinessCheckerPasses(unittest.TestCase): + def test_g2_digest_binds_non_target_corpus_bytes_with_conformance_parity(self) -> None: + with committed_repo() as (root, baseline_commit): + baseline = g2_surface_digest_at_commit(baseline_commit, root) + self.assertEqual( + baseline, + conformance_surface_digest_at_commit(root, baseline_commit), + ) + corpus = root / "spec" / "adapter-conformance.json" + corpus.write_bytes( + corpus.read_bytes().replace( + b' "schema_version": 1,', + b' "schema_version": 1,', + 1, + ) + ) + subprocess.run(("git", "add", str(corpus)), cwd=root, check=True) + subprocess.run( + ("git", "commit", "-q", "-m", "change corpus whitespace"), + cwd=root, check=True, + ) + changed_commit = subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + changed = g2_surface_digest_at_commit(changed_commit, root) + self.assertNotEqual(changed, baseline) + self.assertEqual( + changed, + conformance_surface_digest_at_commit(root, changed_commit), + ) + + @staticmethod + def _complete_g3_report(root: Path, commit: str) -> dict[str, object]: + return { + "kind": "external", + "ref": "https://reviews.example.org/cryptography/report-1", + "producer": "Independent Cryptography Laboratory", + "producer_identity": "https://identity.example.org/crypto-lab", + "observed": "2026-08-12", + "review_type": "production-cryptography", + "reviewed_commit": commit, + "scope": sorted(G3_SCOPE), + "result": "pass-with-findings", + "relationship": "independent-third-party", + "conflicts": [], + "independence_attestation": "llm-errata-independent-cryptography-review-v1", + "surface_digest": gate_surface_digest(root, G3_SURFACE_FILES), + "implementation": { + "library": "example-constant-time-library", + "version": "1.2.3", + "binding": "example-python-binding", + "build_digest": "sha256:" + "a" * 64, + "platforms": ["linux-amd64", "macos-arm64"], + "constant_time": True, + "audited_build": True, + }, + } + + def test_g3_complete_report_qualifies_and_malformed_scope_fails_closed(self) -> None: + with committed_repo() as (root, commit): + report = self._complete_g3_report(root, commit) + self.assertTrue(qualifying_g3_security_evidence(report, root=root)) + for invalid in ([[]], [{}], ["constant-time", []]): + with self.subTest(scope=invalid): + malformed = copy.deepcopy(report) + malformed["scope"] = invalid + self.assertFalse( + qualifying_g3_security_evidence(malformed, root=root) + ) + + @staticmethod + def _g4_common( + root: Path, + commit: str, + *, + producer: str, + producer_identity: str, + role: str, + implementation_id: str, + ) -> dict[str, object]: + return { + "kind": "external", + "ref": f"https://evidence.example.org/g4/{implementation_id}", + "producer": producer, + "producer_identity": producer_identity, + "observed": "2026-08-12", + "review_type": "g4-conformance", + "reviewed_commit": commit, + "result": "pass", + "relationship": ( + "independent-implementation" + if role == "adapter" + else "independent-third-party-validator" + ), + "conflicts": [], + "independence_attestation": "llm-errata-independent-implementation-v1", + "surface_digest": g2_surface_digest(root), + "evidence_role": role, + "implementation_id": implementation_id, + "erratum_id": "erratum-synthetic-001", + } + + @classmethod + def _complete_g4_records( + cls, root: Path, commit: str + ) -> tuple[dict[str, object], dict[str, object], dict[str, object]]: + adapter_a = cls._g4_common( + root, commit, producer="Systems Laboratory Alpha", + producer_identity="https://identity.example.org/lab-alpha", + role="adapter", implementation_id="adapter-alpha", + ) + adapter_a["receipt"] = { + "receipt_id": "receipt-alpha", + "receipt_digest": "sha256:" + "a" * 64, + "evidence_ref": "https://evidence.example.org/receipts/alpha", + } + adapter_b = cls._g4_common( + root, commit, producer="Systems Laboratory Beta", + producer_identity="https://identity.example.org/lab-beta", + role="adapter", implementation_id="adapter-beta", + ) + adapter_b["receipt"] = { + "receipt_id": "receipt-beta", + "receipt_digest": "sha256:" + "b" * 64, + "evidence_ref": "https://evidence.example.org/receipts/beta", + } + validator = cls._g4_common( + root, commit, producer="Independent Validator Laboratory", + producer_identity="https://identity.example.org/validator-lab", + role="validator", implementation_id="validator-one", + ) + validator["validated_receipts"] = [ + { + "implementation_id": "adapter-alpha", + "receipt_id": "receipt-alpha", + "receipt_digest": "sha256:" + "a" * 64, + "result": "pass", + "evidence_ref": "https://evidence.example.org/validation/alpha", + }, + { + "implementation_id": "adapter-beta", + "receipt_id": "receipt-beta", + "receipt_digest": "sha256:" + "b" * 64, + "result": "pass", + "evidence_ref": "https://evidence.example.org/validation/beta", + }, + ] + return adapter_a, adapter_b, validator + + def test_g4_validator_must_bind_both_adapter_receipts_for_same_erratum(self) -> None: + with committed_repo() as (root, commit): + adapter_a, adapter_b, validator = self._complete_g4_records(root, commit) + self.assertTrue(qualifying_g4_evidence( + [adapter_a, adapter_b, validator], root=root + )) + + unrelated = copy.deepcopy(validator) + unrelated["validated_receipts"][0]["implementation_id"] = "adapter-other-a" + unrelated["validated_receipts"][1]["implementation_id"] = "adapter-other-b" + self.assertFalse(qualifying_g4_evidence( + [adapter_a, adapter_b, unrelated], root=root + )) + + wrong_erratum = copy.deepcopy(validator) + wrong_erratum["erratum_id"] = "erratum-unrelated" + self.assertFalse(qualifying_g4_evidence( + [adapter_a, adapter_b, wrong_erratum], root=root + )) + + def test_g4_role_specific_records_reject_malformed_receipt_shapes(self) -> None: + with committed_repo() as (root, commit): + adapter_a, _, validator = self._complete_g4_records(root, commit) + for entry, field, invalid in ( + (adapter_a, "receipt", []), + (adapter_a, "receipt", {"receipt_id": []}), + (validator, "validated_receipts", [[]]), + (validator, "validated_receipts", [{"implementation_id": []}]), + ): + with self.subTest(field=field, invalid=invalid): + malformed = copy.deepcopy(entry) + malformed[field] = invalid + self.assertFalse( + valid_g4_implementation_evidence(malformed, root=root) + ) + + @staticmethod + def _complete_g5_report(root: Path, commit: str) -> dict[str, object]: + metrics = ( + "observation-to-quarantine-time", + "known-descendant-coverage", + "stale-behavior-rate", + "replacement-activation", + "collateral-retention", + "stale-reimport-resistance", + "opaque-coverage", + "operator-effort", + "user-visible-friction", + ) + systems = [] + for index, name in enumerate(("system-alpha", "system-beta", "system-gamma")): + systems.append({ + "name": name, + "version": "1.0.0", + "operator": f"Operator {index + 1}", + "operator_identity": f"https://identity.example.org/operator-{index + 1}", + "evidence_ref": f"https://evidence.example.org/systems/{name}", + "result": "pass", + "independently_operated": True, + "intentionally_nonconforming": index == 0, + "coverage": "opaque" if index == 1 else "complete", + "mixed_artifact_lineage": index == 2, + "operations": [ + { + "operation": operation, + "completed": True, + "evidence_ref": ( + f"https://evidence.example.org/systems/{name}/{operation}" + ), + } + for operation in ("correction", "supersession", "erasure") + ], + "measurements": [ + { + "metric": metric, + "value": index + 1, + "unit": "synthetic-unit", + "evidence_ref": ( + f"https://evidence.example.org/systems/{name}/metrics/{metric}" + ), + } + for metric in metrics + ], + }) + return { + "kind": "external", + "ref": "https://reviews.example.org/interoperability/report-1", + "producer": "Independent Interoperability Laboratory", + "producer_identity": "https://identity.example.org/interoperability-lab", + "observed": "2026-08-12", + "review_type": "phase3-interoperability", + "reviewed_commit": commit, + "surface_digest": gate_surface_digest(root, G5_SURFACE_FILES), + "result": "pass-with-findings", + "relationship": "independent-experiment-report", + "conflicts": [], + "independence_attestation": "llm-errata-independent-interoperability-review-v1", + "synthetic_data": True, + "user_controlled_root": True, + "root_id": "synthetic-root-001", + "systems": systems, + } + + def test_g5_complete_declared_experiment_qualifies(self) -> None: + with committed_repo() as (root, commit): + report = self._complete_g5_report(root, commit) + self.assertTrue(qualifying_g5_interoperability_evidence(report, root=root)) + + def test_g5_partial_or_malformed_experiment_cannot_qualify(self) -> None: + with committed_repo() as (root, commit): + baseline = self._complete_g5_report(root, commit) + legacy_shape = copy.deepcopy(baseline) + legacy_shape.pop("user_controlled_root") + for system in legacy_shape["systems"]: + for field in ( + "intentionally_nonconforming", "coverage", + "mixed_artifact_lineage", "operations", "measurements", + ): + system.pop(field) + self.assertFalse( + qualifying_g5_interoperability_evidence(legacy_shape, root=root) + ) + mutations = { + "missing operation": lambda report: report["systems"][0]["operations"].pop(), + "missing measurement": lambda report: report["systems"][0]["measurements"].pop(), + "no nonconforming importer": lambda report: [ + system.update(intentionally_nonconforming=False) + for system in report["systems"] + ], + "no incomplete system": lambda report: [ + system.update(coverage="complete") for system in report["systems"] + ], + "no mixed lineage": lambda report: [ + system.update(mixed_artifact_lineage=False) + for system in report["systems"] + ], + "malformed system": lambda report: report.update(systems=[[]]), + } + for name, mutate in mutations.items(): + with self.subTest(name=name): + report = copy.deepcopy(baseline) + mutate(report) + self.assertFalse( + qualifying_g5_interoperability_evidence(report, root=root) + ) + def test_cryptography_qualification_foregrounds_refusal_evidence(self) -> None: qualification = ( Path(__file__).resolve().parents[1] @@ -90,6 +420,7 @@ def test_g2_surface_includes_checkpoint_contract_and_tests(self) -> None: self.assertIn("prototype/checkpoints.py", files) self.assertIn("tests/test_checkpoints.py", files) self.assertIn("spec/adapter-conformance.json", files) + self.assertIn("docs/READINESS_EVIDENCE_SCHEMAS.md", files) def test_g6_complete_measured_report_is_commit_and_deployment_bound(self) -> None: with repo_copy() as source, tempfile.TemporaryDirectory() as temp: From d396e9515e1f82bc6da1abfc8fb19b7cdf073019 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 05:20:23 +0200 Subject: [PATCH 5/8] fix: enforce v3 publication lineage --- scripts/check_publication.py | 11 +++++++---- tests/test_publication.py | 8 ++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/check_publication.py b/scripts/check_publication.py index ae6ecda..8dd6687 100644 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -165,7 +165,7 @@ def _manifest_history() -> tuple[list[dict[str, object]], list[str]]: f"publication history: manifest at {commit[:12]} lacks surface records" ) continue - if payload.get("schema_version") == 1: + if payload.get("schema_version") in {1, 2}: continue if not isinstance(payload.get("historical_surfaces"), list): failures.append( @@ -352,10 +352,13 @@ def validate_manifest(payload: object) -> list[str]: failures.append("review target: runtime surface differs from reviewed source") except OSError as error: failures.append(f"review target: {error}") - changed = _git("diff", "--name-only", f"{commit}..HEAD") - if changed.returncode != 0: + changed = _git("diff", "--name-only", commit) + untracked = _git("ls-files", "--others", "--exclude-standard") + if changed.returncode != 0 or untracked.returncode != 0: failures.append("release binding: packaging delta cannot be inspected") - elif set(changed.stdout.splitlines()) - ALLOWED_PACKAGING_PATHS: + elif ( + set(changed.stdout.splitlines()) | set(untracked.stdout.splitlines()) + ) - ALLOWED_PACKAGING_PATHS: failures.append("release binding: runtime contains non-packaging changes after review target") return failures diff --git a/tests/test_publication.py b/tests/test_publication.py index db6a5cd..9d4f3ef 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -218,6 +218,14 @@ def mutate(payload: dict[str, object]) -> None: self._assert_history_mutation_is_rejected(mutate, "active surfaces") + def test_uncommitted_nonpackaging_delta_is_rejected(self) -> None: + with bound_publication_repo() as root: + readme = root / "README.md" + readme.write_bytes(readme.read_bytes() + b"\nnon-packaging mutation\n") + result = run_checker(root, SCRIPT) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout + result.stderr) + self.assertIn("release binding", result.stdout) + def test_duplicate_surface_url_is_rejected(self) -> None: def mutate(payload: dict[str, object]) -> None: payload["surfaces"].append(dict(payload["surfaces"][0])) From 27b7677e7b9936506ca64e84c64fc0272efa5969 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 05:22:15 +0200 Subject: [PATCH 6/8] docs: bind final v0.4.1 review target --- publication/active-surfaces.json | 43 ++++++++++++++++++++++++++++---- spec/adapter-conformance.json | 4 +-- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/publication/active-surfaces.json b/publication/active-surfaces.json index 032d3c3..adbe1bf 100644 --- a/publication/active-surfaces.json +++ b/publication/active-surfaces.json @@ -1,8 +1,8 @@ { "schema_version": 3, "review_target": { - "commit": "ae9018c5d464b8ddd0fdf2ac99577e5e7fdc562b", - "surface_digest": "fc2be2c194c801349c4e39462394b6e1e52b2e479619b68a5615e7386bfdf5b9" + "commit": "d396e9515e1f82bc6da1abfc8fb19b7cdf073019", + "surface_digest": "a5a4152cd679f21a903f4e36ad19f59f4303bf47e089127030f667975cc26a2d" }, "release_binding": { "version": "0.4.1", @@ -74,9 +74,7 @@ ], "evidence_boundary": "recruitment-only", "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288798624" - } - ], - "surfaces": [ + }, { "id": "v041-corrected-review-target", "kind": "issue-comment", @@ -90,6 +88,41 @@ "supersedes": [ "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288780760" ], + "evidence_boundary": "recruitment-only", + "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288996319" + }, + { + "id": "v041-second-corrected-review-target", + "kind": "issue-comment", + "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288996319", + "published": "2026-08-14", + "commit": "504cb659436f9d719aef27158191ced8422af022", + "surface_digest": "a5a4152cd679f21a903f4e36ad19f59f4303bf47e089127030f667975cc26a2d", + "gates": ["G2", "G4"], + "roles": ["independent-reviewer"], + "mentions": [], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288798624" + ], + "evidence_boundary": "recruitment-only", + "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5289016713" + } + ], + "surfaces": [ + { + "id": "v041-final-corrected-review-target", + "kind": "issue-comment", + "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5289016713", + "published": "2026-08-14", + "commit": "d396e9515e1f82bc6da1abfc8fb19b7cdf073019", + "surface_digest": "a5a4152cd679f21a903f4e36ad19f59f4303bf47e089127030f667975cc26a2d", + "gates": ["G2", "G4"], + "roles": ["independent-reviewer"], + "mentions": [], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288996319", + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288798624" + ], "evidence_boundary": "recruitment-only" } ] diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json index 5b359a5..7a18681 100644 --- a/spec/adapter-conformance.json +++ b/spec/adapter-conformance.json @@ -3,8 +3,8 @@ "status": "candidate-internal", "evidence_boundary": "Passing this corpus is internal conformance evidence. It is not G2 or G4 evidence.", "normative_target": { - "commit": "ae9018c5d464b8ddd0fdf2ac99577e5e7fdc562b", - "surface_digest": "fc2be2c194c801349c4e39462394b6e1e52b2e479619b68a5615e7386bfdf5b9" + "commit": "d396e9515e1f82bc6da1abfc8fb19b7cdf073019", + "surface_digest": "a5a4152cd679f21a903f4e36ad19f59f4303bf47e089127030f667975cc26a2d" }, "provenance": { "reported_by": "Rastislav Drahos / DanceNitra", From 485f95a018e750d8976be9df29ed31491e0d9a05 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 05:51:28 +0200 Subject: [PATCH 7/8] fix: close readiness and publication review gaps --- docs/READINESS_EVIDENCE_SCHEMAS.md | 12 ++- scripts/check_publication.py | 60 ++++++++++++++ scripts/check_readiness.py | 124 ++++++++++++++++++----------- tests/test_publication.py | 49 ++++++++++++ tests/test_readiness.py | 82 +++++++++++++++++++ 5 files changed, 280 insertions(+), 47 deletions(-) diff --git a/docs/READINESS_EVIDENCE_SCHEMAS.md b/docs/READINESS_EVIDENCE_SCHEMAS.md index d276fd7..5a6588f 100644 --- a/docs/READINESS_EVIDENCE_SCHEMAS.md +++ b/docs/READINESS_EVIDENCE_SCHEMAS.md @@ -9,6 +9,9 @@ Shared fields are `kind=external`, a public report `ref`, non-empty `producer`, public `producer_identity`, ISO `observed` date, exact `reviewed_commit`, exact `surface_digest`, `conflicts` array, gate-specific `review_type`, relationship, and independence attestation. Extra or malformed fields fail closed. +`result` is exactly `pass`, `pass-with-findings`, or `fail`. Structural +admission preserves a complete negative report; gate qualification separately +requires the passing values described below. ## G3 cryptography record @@ -38,7 +41,8 @@ producer identity, and `validated_receipts`. Every result names the adapter reference. G4 qualifies only when one validator reports passing results for the exact receipts from two different adapter producers against the same erratum, commit, and surface digest. An unrelated validator report cannot be -combined with the adapters. +combined with the adapters. Structurally complete adapter, validator, and +per-receipt `fail` results remain admissible evidence but do not qualify G4. ## G5 interoperability record @@ -63,7 +67,11 @@ operator identities. Every system names version and evidence, completes Every operation and measurement has its own public evidence reference. The three-system set must include at least one intentionally nonconforming importer, one `incomplete` or `opaque` coverage system, and one mixed-artifact-lineage -system. Three generic passing system summaries are insufficient. +system. A structurally complete failed report may record a failed system, +`independently_operated=false`, or `completed=false`. Qualification requires a +passing report, synthetic data, a user-controlled root, three independently +operated passing systems, every operation completed, and all three required +experiment arms. Three generic passing system summaries are insufficient. These schemas establish structural admission only. Maintainer review still must verify real identities, authorship, conflicts, authorization, raw evidence, diff --git a/scripts/check_publication.py b/scripts/check_publication.py index 8dd6687..fd92f3d 100644 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -197,6 +197,65 @@ def _historical_version_of(active: object, historical: object) -> bool: ) +def _validate_supersession_chains( + active: list[object], historical: list[object] +) -> list[str]: + """Require every historical forward link to have an exact reverse link.""" + + records = [record for record in (*historical, *active) if isinstance(record, dict)] + by_url = { + record["url"]: record + for record in records + if isinstance(record.get("url"), str) + } + active_urls = { + record["url"] + for record in active + if isinstance(record, dict) and isinstance(record.get("url"), str) + } + if len(by_url) != len(records): + return ["active surfaces: supersession-chain URLs must be unique"] + + failed = False + for origin in historical: + if not isinstance(origin, dict): + continue + origin_url = origin.get("url") + successor_url = origin.get("superseded_by") + successor = by_url.get(successor_url) + if not ( + isinstance(origin_url, str) + and isinstance(successor, dict) + and isinstance(successor.get("supersedes"), list) + and origin_url in successor["supersedes"] + ): + failed = True + continue + + visited = {origin_url} + cursor = successor + while cursor.get("url") not in active_urls: + cursor_url = cursor.get("url") + next_url = cursor.get("superseded_by") + if not isinstance(cursor_url, str) or cursor_url in visited: + failed = True + break + visited.add(cursor_url) + next_record = by_url.get(next_url) + if not ( + isinstance(next_record, dict) + and isinstance(next_record.get("supersedes"), list) + and cursor_url in next_record["supersedes"] + ): + failed = True + break + cursor = next_record + + return [ + "active surfaces: supersession chains must be bidirectional and terminate at a current surface" + ] if failed else [] + + def _validate_append_only_history(payload: dict[str, object]) -> list[str]: """Require every committed surface transition to preserve exact prior records.""" @@ -338,6 +397,7 @@ def validate_manifest(payload: object) -> list[str]: failures.append("unique evidence roles: each active role needs one owner") if len(mentions) != len(set(mentions)): failures.append("unique GitHub mentions: each identity may be notified only once") + failures.extend(_validate_supersession_chains(surfaces, historical)) if (ROOT / ".git").exists() and _valid_target(target): failures.extend(_validate_append_only_history(payload)) diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 7b2ed7e..19344e6 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -221,6 +221,12 @@ def valid_g2_identity_ref(value: object) -> bool: return parsed.scheme == "https" and bool(parsed.netloc) and parsed.path not in {"", "/"} +def _token_in(value: object, allowed: set[str] | frozenset[str]) -> bool: + """Test enum membership without hashing malformed JSON containers.""" + + return isinstance(value, str) and value in allowed + + def g2_surface_files(root: Path = ROOT) -> tuple[str, ...]: """Return the comprehensive sorted first-party Phase 2 review manifest.""" @@ -291,7 +297,7 @@ def valid_g2_review_evidence( and all(isinstance(token, str) for token in scope) and len(scope) == len(set(scope)) and set(scope) == G2_SCOPE - and entry.get("result") in G2_RESULTS + and _token_in(entry.get("result"), G2_RESULTS) and entry.get("relationship") == "independent-third-party" and isinstance(entry.get("conflicts"), list) and isinstance(entry.get("producer_identity"), str) @@ -309,10 +315,9 @@ def qualifying_g2_review_evidence( ) -> bool: """Return whether a valid G2 review can satisfy a G2 PASS gate.""" - return valid_g2_review_evidence(entry, today=today, root=root) and entry.get("result") in { - "pass", - "pass-with-findings", - } + return valid_g2_review_evidence(entry, today=today, root=root) and _token_in( + entry.get("result"), {"pass", "pass-with-findings"} + ) def _surface_digest_for_files(files: tuple[str, ...], root: Path) -> str: @@ -368,14 +373,14 @@ def _commit_bound_external( and valid_g2_report_ref(entry.get("ref")) and isinstance(commit, str) and re.fullmatch(r"[0-9a-f]{40}", commit) is not None - and entry.get("relationship") in { + and _token_in(entry.get("relationship"), { "independent-third-party", "independent-implementation", "independent-third-party-validator", "independent-experiment-report", - } + }) and isinstance(entry.get("conflicts"), list) and valid_g2_identity_ref(entry.get("producer_identity")) and entry.get("independence_attestation") == attestation - and entry.get("result") in G2_RESULTS + and _token_in(entry.get("result"), G2_RESULTS) ): return False try: @@ -390,7 +395,7 @@ def _commit_bound_external( return False -def qualifying_g3_security_evidence( +def valid_g3_security_evidence( entry: object, *, today: date | None = None, root: Path = ROOT ) -> bool: if not _commit_bound_external( @@ -404,7 +409,6 @@ def qualifying_g3_security_evidence( set(entry) == G3_ENTRY_KEYS and entry.get("review_type") == "production-cryptography" and entry.get("relationship") == "independent-third-party" - and entry.get("result") in {"pass", "pass-with-findings"} and isinstance(scope, list) and all(isinstance(token, str) for token in scope) and len(scope) == len(G3_SCOPE) @@ -419,8 +423,19 @@ def qualifying_g3_security_evidence( and re.fullmatch(r"sha256:[0-9a-f]{64}", implementation["build_digest"]) is not None and isinstance(implementation["platforms"], list) and bool(implementation["platforms"]) and all(_nonempty(item) for item in implementation["platforms"]) - and implementation["constant_time"] is True - and implementation["audited_build"] is True + and isinstance(implementation["constant_time"], bool) + and isinstance(implementation["audited_build"], bool) + ) + + +def qualifying_g3_security_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: + return ( + valid_g3_security_evidence(entry, today=today, root=root) + and _token_in(entry["result"], {"pass", "pass-with-findings"}) + and entry["implementation"]["constant_time"] is True + and entry["implementation"]["audited_build"] is True ) @@ -439,9 +454,8 @@ def valid_g4_implementation_evidence( ) if not ( entry.get("review_type") == "g4-conformance" - and role in {"adapter", "validator"} + and _token_in(role, {"adapter", "validator"}) and entry.get("relationship") == expected_relationship - and entry.get("result") in {"pass", "pass-with-findings"} and _nonempty(entry.get("implementation_id")) and _nonempty(entry.get("erratum_id")) ): @@ -472,7 +486,7 @@ def valid_g4_implementation_evidence( and isinstance(result["receipt_digest"], str) and re.fullmatch(r"sha256:[0-9a-f]{64}", result["receipt_digest"]) is not None - and result["result"] in {"pass", "pass-with-findings"} + and _token_in(result["result"], G2_RESULTS) and valid_g2_report_ref(result["evidence_ref"]) ): return False @@ -488,8 +502,16 @@ def qualifying_g4_evidence( entry for entry in entries if valid_g4_implementation_evidence(entry, today=today, root=root) ] - adapters = [entry for entry in valid if entry["evidence_role"] == "adapter"] - validators = [entry for entry in valid if entry["evidence_role"] == "validator"] + adapters = [ + entry for entry in valid + if entry["evidence_role"] == "adapter" + and _token_in(entry["result"], {"pass", "pass-with-findings"}) + ] + validators = [ + entry for entry in valid + if entry["evidence_role"] == "validator" + and _token_in(entry["result"], {"pass", "pass-with-findings"}) + ] if len(adapters) < 2 or not validators: return False all_adapter_identities = {entry["producer_identity"] for entry in adapters} @@ -502,7 +524,7 @@ def qualifying_g4_evidence( result["receipt_digest"], ) for result in validator["validated_receipts"] - if result["result"] in {"pass", "pass-with-findings"} + if _token_in(result["result"], {"pass", "pass-with-findings"}) } validator_target = (validator["reviewed_commit"], validator["surface_digest"]) for first, second in combinations(adapters, 2): @@ -532,7 +554,7 @@ def qualifying_g4_evidence( return False -def qualifying_g5_interoperability_evidence( +def valid_g5_interoperability_evidence( entry: object, *, today: date | None = None, root: Path = ROOT ) -> bool: if not _commit_bound_external( @@ -545,9 +567,8 @@ def qualifying_g5_interoperability_evidence( set(entry) == G5_ENTRY_KEYS and entry.get("review_type") == "phase3-interoperability" and entry.get("relationship") == "independent-experiment-report" - and entry.get("result") in {"pass", "pass-with-findings"} - and entry.get("synthetic_data") is True - and entry.get("user_controlled_root") is True + and isinstance(entry.get("synthetic_data"), bool) + and isinstance(entry.get("user_controlled_root"), bool) and _nonempty(entry.get("root_id")) and isinstance(systems, list) and len(systems) == 3 ): @@ -558,10 +579,10 @@ def qualifying_g5_interoperability_evidence( and all(_nonempty(system[key]) for key in ("name", "version", "operator")) and valid_g2_identity_ref(system["operator_identity"]) and valid_g2_report_ref(system["evidence_ref"]) - and system["result"] == "pass" - and system["independently_operated"] is True + and _token_in(system["result"], {"pass", "fail"}) + and isinstance(system["independently_operated"], bool) and isinstance(system["intentionally_nonconforming"], bool) - and system["coverage"] in {"complete", "incomplete", "opaque"} + and _token_in(system["coverage"], {"complete", "incomplete", "opaque"}) and isinstance(system["mixed_artifact_lineage"], bool) ): return False @@ -572,7 +593,7 @@ def qualifying_g5_interoperability_evidence( and all(_exact_dict(operation, G5_OPERATION_KEYS) for operation in operations) and all(isinstance(operation["operation"], str) for operation in operations) and {operation["operation"] for operation in operations} == G5_OPERATIONS - and all(operation["completed"] is True for operation in operations) + and all(isinstance(operation["completed"], bool) for operation in operations) and all(valid_g2_report_ref(operation["evidence_ref"]) for operation in operations) ): return False @@ -591,6 +612,26 @@ def qualifying_g5_interoperability_evidence( return ( len({system["name"] for system in systems}) == 3 and len({system["operator_identity"] for system in systems}) == 3 + ) + + +def qualifying_g5_interoperability_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: + if not valid_g5_interoperability_evidence(entry, today=today, root=root): + return False + systems = entry["systems"] + return ( + _token_in(entry["result"], {"pass", "pass-with-findings"}) + and entry["synthetic_data"] is True + and entry["user_controlled_root"] is True + and all(system["result"] == "pass" for system in systems) + and all(system["independently_operated"] is True for system in systems) + and all( + operation["completed"] is True + for system in systems + for operation in system["operations"] + ) and any(system["intentionally_nonconforming"] for system in systems) and any(system["coverage"] in {"incomplete", "opaque"} for system in systems) and any(system["mixed_artifact_lineage"] for system in systems) @@ -611,24 +652,11 @@ def g6_surface_files(root: Path = ROOT) -> tuple[str, ...]: def g6_surface_digest(root: Path = ROOT) -> str: - return surface_digest_from_bytes( - [(relative, (root / relative).read_bytes()) for relative in g6_surface_files(root)] - ) + return _surface_digest_for_files(g6_surface_files(root), root) def g6_surface_digest_at_commit(commit: str, root: Path = ROOT) -> str: - if not reviewed_commit_exists(commit, root): - raise OSError("reviewed commit is unavailable") - entries = [] - for relative in g6_surface_files(root): - result = subprocess.run( - ["git", "show", f"{commit}:{relative}"], cwd=root, - capture_output=True, check=False, - ) - if result.returncode != 0: - raise OSError(f"reviewed commit lacks {relative}") - entries.append((relative, result.stdout)) - return surface_digest_from_bytes(entries) + return _surface_digest_for_files_at_commit(g6_surface_files(root), commit, root) def _exact_dict(value: object, keys: set[str]) -> bool: @@ -660,7 +688,7 @@ def _valid_measurement(value: object) -> bool: _nonempty(value["metric"]) and _finite_number(value["value"]) and _nonempty(value["unit"]) - and value["comparator"] in COMPARATORS + and _token_in(value["comparator"], set(COMPARATORS)) and _finite_number(value["threshold"]) and valid_g2_report_ref(value["evidence_ref"]) ) @@ -690,7 +718,7 @@ def valid_g6_operational_evidence( and entry.get("relationship") == "independent-third-party" and isinstance(entry.get("conflicts"), list) and entry.get("independence_attestation") == G6_ATTESTATION - and entry.get("result") in G2_RESULTS + and _token_in(entry.get("result"), G2_RESULTS) and isinstance(reviewed_commit, str) and re.fullmatch(r"[0-9a-f]{40}", reviewed_commit) is not None and _exact_dict(deployment, { @@ -725,7 +753,7 @@ def valid_g6_operational_evidence( measurements = scope["measurements"] if not ( isinstance(scope["scope"], str) - and scope["status"] in {"pass", "fail"} + and _token_in(scope["status"], {"pass", "fail"}) and isinstance(artifacts, list) and artifacts and all(valid_g2_report_ref(item) for item in artifacts) and isinstance(measurements, list) and measurements @@ -751,7 +779,7 @@ def qualifying_g6_operational_evidence( if not valid_g6_operational_evidence(entry, today=today, root=root): return False return ( - entry["result"] in {"pass", "pass-with-findings"} + _token_in(entry["result"], {"pass", "pass-with-findings"}) and all(scope["status"] == "pass" for scope in entry["scopes"]) and all( _measurement_passes(measurement) @@ -1101,6 +1129,12 @@ def validate_ledger( entry_valid = valid_external_evidence(entry) if gate_id == "G2": entry_valid = valid_g2_review_evidence(entry, root=ROOT) + elif gate_id == "G3": + entry_valid = valid_g3_security_evidence(entry, root=ROOT) + elif gate_id == "G4": + entry_valid = valid_g4_implementation_evidence(entry, root=ROOT) + elif gate_id == "G5": + entry_valid = valid_g5_interoperability_evidence(entry, root=ROOT) elif gate_id == "G6": entry_valid = valid_g6_operational_evidence(entry, root=ROOT) evidence_valid = evidence_valid and entry_valid diff --git a/tests/test_publication.py b/tests/test_publication.py index 9d4f3ef..433c73b 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -218,6 +218,55 @@ def mutate(payload: dict[str, object]) -> None: self._assert_history_mutation_is_rejected(mutate, "active surfaces") + def test_superseded_by_must_point_to_the_replacement_surface(self) -> None: + def mutate(payload: dict[str, object]) -> None: + prior = payload["surfaces"].pop() + historical = dict(prior) + historical["superseded_by"] = ( + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-9999999997" + ) + payload["historical_surfaces"].append(historical) + replacement = dict(prior) + replacement["id"] = "replacement-current-surface" + replacement["url"] = ( + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-9999999996" + ) + replacement["supersedes"] = [prior["url"]] + payload["surfaces"].append(replacement) + + self._assert_history_mutation_is_rejected(mutate, "active surfaces") + + def test_non_ancestor_review_target_is_rejected(self) -> None: + with bound_publication_repo() as root: + tree = subprocess.run( + ("git", "rev-parse", "HEAD^{tree}"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + unrelated = subprocess.run( + ("git", "commit-tree", tree), cwd=root, check=True, + input="unrelated source\n", capture_output=True, text=True, + ).stdout.strip() + digest = g2_surface_digest_at_commit(unrelated, root) + publication_path = root / "publication" / "active-surfaces.json" + corpus_path = root / "spec" / "adapter-conformance.json" + publication = json.loads(publication_path.read_text(encoding="utf-8")) + corpus = json.loads(corpus_path.read_text(encoding="utf-8")) + target = {"commit": unrelated, "surface_digest": digest} + publication["review_target"] = target + corpus["normative_target"] = target + for surface in publication["surfaces"]: + surface["commit"] = unrelated + surface["surface_digest"] = digest + publication_path.write_text( + json.dumps(publication, indent=2) + "\n", encoding="utf-8" + ) + corpus_path.write_text( + json.dumps(corpus, indent=2) + "\n", encoding="utf-8" + ) + result = run_checker(root, SCRIPT) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout + result.stderr) + self.assertIn("commit must be an ancestor", result.stdout) + def test_uncommitted_nonpackaging_delta_is_rejected(self) -> None: with bound_publication_repo() as root: readme = root / "README.md" diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 5cd91ef..4b67051 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -29,7 +29,9 @@ qualifying_g2_review_evidence, valid_external_evidence, valid_g2_review_evidence, + valid_g3_security_evidence, valid_g4_implementation_evidence, + valid_g5_interoperability_evidence, valid_g6_operational_evidence, markdown_value, surface_digest_from_bytes, @@ -356,6 +358,71 @@ def test_g5_complete_declared_experiment_qualifies(self) -> None: report = self._complete_g5_report(root, commit) self.assertTrue(qualifying_g5_interoperability_evidence(report, root=root)) + def test_failed_g3_g4_g5_records_are_valid_but_do_not_qualify(self) -> None: + with committed_repo() as (root, commit): + g3 = self._complete_g3_report(root, commit) + g3["result"] = "fail" + g3["implementation"]["constant_time"] = False + self.assertTrue(valid_g3_security_evidence(g3, root=root)) + self.assertFalse(qualifying_g3_security_evidence(g3, root=root)) + + adapter_a, adapter_b, validator = self._complete_g4_records(root, commit) + adapter_a["result"] = "fail" + validator["validated_receipts"][0]["result"] = "fail" + self.assertTrue(valid_g4_implementation_evidence(adapter_a, root=root)) + self.assertTrue(valid_g4_implementation_evidence(validator, root=root)) + self.assertFalse( + qualifying_g4_evidence([adapter_a, adapter_b, validator], root=root) + ) + + g5 = self._complete_g5_report(root, commit) + g5["result"] = "fail" + g5["systems"][0]["result"] = "fail" + g5["systems"][0]["operations"][0]["completed"] = False + self.assertTrue(valid_g5_interoperability_evidence(g5, root=root)) + self.assertFalse(qualifying_g5_interoperability_evidence(g5, root=root)) + + def test_external_evidence_unhashable_status_tokens_fail_closed(self) -> None: + with committed_repo() as (root, commit): + g2 = self._complete_review(root, commit) + g3 = self._complete_g3_report(root, commit) + adapter, _, validator = self._complete_g4_records(root, commit) + g5 = self._complete_g5_report(root, commit) + g6 = self._complete_g6_report(root, commit) + cases = ( + ("G2 result", g2, lambda record: record.update(result=[]), + valid_g2_review_evidence), + ("G3 result", g3, lambda record: record.update(result=[]), + valid_g3_security_evidence), + ("G4 result", adapter, lambda record: record.update(result={}), + valid_g4_implementation_evidence), + ("G4 receipt result", validator, + lambda record: record["validated_receipts"][0].update(result=[]), + valid_g4_implementation_evidence), + ("G5 result", g5, lambda record: record.update(result={}), + valid_g5_interoperability_evidence), + ("G5 system result", g5, + lambda record: record["systems"][0].update(result=[]), + valid_g5_interoperability_evidence), + ("G5 coverage", g5, + lambda record: record["systems"][0].update(coverage={}), + valid_g5_interoperability_evidence), + ("G6 result", g6, lambda record: record.update(result=[]), + valid_g6_operational_evidence), + ("G6 scope status", g6, + lambda record: record["scopes"][0].update(status={}), + valid_g6_operational_evidence), + ("G6 comparator", g6, + lambda record: record["scopes"][0]["measurements"][0].update( + comparator=[] + ), valid_g6_operational_evidence), + ) + for name, baseline, mutate, validator_fn in cases: + with self.subTest(name=name): + malformed = copy.deepcopy(baseline) + mutate(malformed) + self.assertFalse(validator_fn(malformed, root=root)) + def test_g5_partial_or_malformed_experiment_cannot_qualify(self) -> None: with committed_repo() as (root, commit): baseline = self._complete_g5_report(root, commit) @@ -977,6 +1044,21 @@ def mutate(payload, selected=gate_id): result, f"{gate_id} external PASS evidence" ) + def test_blocked_g3_g4_g5_reject_malformed_independent_records(self) -> None: + for gate_id in ("G3", "G4", "G5"): + with self.subTest(gate_id=gate_id): + def mutate(payload, selected=gate_id): + gate = next(g for g in payload["gates"] if g["id"] == selected) + gate["evidence"].append({ + "kind": "external", + "ref": f"https://reviews.example.org/{selected.lower()}/malformed", + "producer": "Independent Evidence Laboratory", + "observed": "2026-08-14", + }) + + result = self._mutated(mutate) + self.assert_rejected_without_traceback(result, f"{gate_id} evidence") + def test_pre_corpus_review_target_fails_explicitly(self) -> None: with repo_copy() as root: corpus = root / "spec" / "adapter-conformance.json" From 2a818baceecbe6ad65f900bf1eff0536041094e6 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 05:55:23 +0200 Subject: [PATCH 8/8] docs: bind post-review v0.4.1 target --- publication/active-surfaces.json | 26 +++++++++++++++++++++----- spec/adapter-conformance.json | 4 ++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/publication/active-surfaces.json b/publication/active-surfaces.json index adbe1bf..f87df75 100644 --- a/publication/active-surfaces.json +++ b/publication/active-surfaces.json @@ -1,8 +1,8 @@ { "schema_version": 3, "review_target": { - "commit": "d396e9515e1f82bc6da1abfc8fb19b7cdf073019", - "surface_digest": "a5a4152cd679f21a903f4e36ad19f59f4303bf47e089127030f667975cc26a2d" + "commit": "485f95a018e750d8976be9df29ed31491e0d9a05", + "surface_digest": "01072558426c9751786b25a879fa2f15195794d625ff7c2adad0c2d854a8e8ec" }, "release_binding": { "version": "0.4.1", @@ -106,9 +106,7 @@ ], "evidence_boundary": "recruitment-only", "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5289016713" - } - ], - "surfaces": [ + }, { "id": "v041-final-corrected-review-target", "kind": "issue-comment", @@ -123,6 +121,24 @@ "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288996319", "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5288798624" ], + "evidence_boundary": "recruitment-only", + "superseded_by": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5289245081" + } + ], + "surfaces": [ + { + "id": "v041-postreview-review-target", + "kind": "issue-comment", + "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5289245081", + "published": "2026-08-14", + "commit": "485f95a018e750d8976be9df29ed31491e0d9a05", + "surface_digest": "01072558426c9751786b25a879fa2f15195794d625ff7c2adad0c2d854a8e8ec", + "gates": ["G2", "G4"], + "roles": ["independent-reviewer"], + "mentions": [], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5289016713" + ], "evidence_boundary": "recruitment-only" } ] diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json index 7a18681..99ed9a8 100644 --- a/spec/adapter-conformance.json +++ b/spec/adapter-conformance.json @@ -3,8 +3,8 @@ "status": "candidate-internal", "evidence_boundary": "Passing this corpus is internal conformance evidence. It is not G2 or G4 evidence.", "normative_target": { - "commit": "d396e9515e1f82bc6da1abfc8fb19b7cdf073019", - "surface_digest": "a5a4152cd679f21a903f4e36ad19f59f4303bf47e089127030f667975cc26a2d" + "commit": "485f95a018e750d8976be9df29ed31491e0d9a05", + "surface_digest": "01072558426c9751786b25a879fa2f15195794d625ff7c2adad0c2d854a8e8ec" }, "provenance": { "reported_by": "Rastislav Drahos / DanceNitra",