diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index f8abdd1..c4c95c6 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -24,6 +24,10 @@ jobs: steps: - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1, Node 24 + with: + # Conformance binds historical immutable normative commits. A shallow + # checkout cannot verify those Git objects and must not run the suite. + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0, Node 24 diff --git a/CHANGELOG.md b/CHANGELOG.md index 13571b9..39a939f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,44 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), Changes to the bounded novelty statement or the source comparison are recorded here even when they narrow or retire part of the claim. That is the intended direction of travel, not an exception. -## [Unreleased](https://github.com/thomaswillner/llm-errata/compare/v0.3.0...HEAD) +## [Unreleased](https://github.com/thomaswillner/llm-errata/compare/v0.4.0...HEAD) + +## [0.4.0](https://github.com/thomaswillner/llm-errata/releases/tag/v0.4.0) - 2026-08-13 + +### Added — adapter conformance and validator hardening + +- Added five provider-neutral adapter cases with complete checkpoint, + aggregate, triad, store, and receipt outcomes. Calls are traced through the + exact target adapter instance rather than matched globally by function name. +- Added bounded proposition-multiplicity preservation. Synthetic conformance + bindings expose stable provider-local proposition identity and active count; + an unobservable count is `unknown`, not a pass. +- Added exact semantic mutation controls. A mutation must complete and produce + its declared counter-result; an exception or different failure cannot earn + credit. +- Added three executable validator attacks covering empty receipts, incomplete + no-op feed acceptance, and constant-`unknown` semantic aggregation. +- Added `errata adapter-conformance`, canonical JSON reporting, and distinct + pass, failed-control, and invalid-evidence exit codes. + +Rastislav Drahos/DanceNitra materially advanced this release by reporting the +adapter-coverage gap, a real duplicate-preservation defect, candidate adapter +behaviors, and the three anti-vacuity attacks. The immutable MIT-licensed source +artifact is +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`; Inspeximus is a G4 +candidate and the source commit discloses Claude Opus 5 co-authorship, so the +input remains interested-party evidence. LLM Errata independently authored its +corpus and validator without copying or vendoring that runner or fixture. + +### Status and ownership + +- Repository files authored here remain copyright Thomas Rainer Willner. + Contributor credit and MIT permission do not transfer copyright. +- Existing licence terms are unchanged: independent commercial and + non-commercial specification implementations are permitted under the + attributed implementation grant; Reference Code remains restricted. +- Verdict remains `NOT_PROD_READY`. G2 through G6 remain blocked by their + existing external evidence requirements. ### Added — production-readiness evidence diff --git a/CITATION.cff b/CITATION.cff index 41ad2e7..b4eb0d7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,8 +5,8 @@ type: software authors: - family-names: "Willner" given-names: "Thomas Rainer" -version: 0.3.0 -date-released: 2026-08-07 +version: 0.4.0 +date-released: 2026-08-13 license: LicenseRef-LLM-Errata-Personal-Use license-url: "https://github.com/thomaswillner/llm-errata/blob/main/LICENSE" abstract: >- @@ -21,4 +21,3 @@ keywords: - provenance - conformance - verifiable repair - diff --git a/IDEA.md b/IDEA.md index c461649..f672b37 100644 --- a/IDEA.md +++ b/IDEA.md @@ -74,6 +74,13 @@ Every repair uses a three-way check—the **repair triad**: 2. **Positive:** the replacement influenced the cases in which it should. 3. **Preservation:** nearby facts that were not changed still worked. +Within an inspectable adapter scope, preservation also requires bounded +proposition multiplicity: repair must not increase active assertions of a +preserved proposition unless the erratum requires another assertion. The +adapter supplies stable provider-local proposition identity and count for the +synthetic conformance fixture. Text similarity is not identity; an adapter +that cannot expose this observation reports it as `unknown`. + Erasure has no positive replacement, but still needs negative and preservation checks. The triad defeats two cheap tricks: adding a new fact while still retrieving the old one, and “fixing” the problem by wiping the whole profile. ## Three operations, three meanings diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index 9d3a3f8..d52af3a 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -60,6 +60,14 @@ the adapter cannot retire one input while preserving the others without an additional decomposition or rebuild contract; it must report that limit rather than fabricate complete repair. +Run the adapter corpus against an isolated synthetic store with +`errata adapter-conformance --binding module:factory`. The binding must expose +stable provider-local proposition identities and active counts for that +fixture. A store unable to observe multiplicity reports `unknown`; text +normalization or embedding similarity cannot substitute for identity. A pass +is candidate internal evidence only and still requires a separate producer to +validate both implementations and the validator. + ## Independence and evidence An implementation report must name its authors, repository and commit, supported diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 30696f2..edcb913 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -4,7 +4,7 @@ | Field | Value | |---|---| -| Version | 0.3.0 | +| Version | 0.4.0 | | Verdict | **NOT_PROD_READY** | | Ledger | `readiness/production-readiness.json` | @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, adapter-level conformance, validator anti-vacuity controls, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | Inspeximus `v2.7.0` is one tagged externally authored adapter candidate with disclosed v2.6.1 reference-code contamination and a claimed clean-room rewrite. It targets historical commit `a477fe4f5c86730031b6285d9505778fb8eec060`; provenance, current-target behavior, a second candidate, and a third-party validator result remain unverified. | Rebind candidates to the current immutable target; obtain dated evidence from two independently authored adapters, including separate provenance review where needed, and a separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/README.md b/README.md index e00561d..22f7c92 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ | Field | Value | |---|---| | Author | Thomas Rainer Willner | -| Version | 0.3.0 | +| Version | 0.4.0 | | Status | Public concept proposal / Request for Comment | | Published | 2026-08-07 | | Research reviewed through | 2026-08-01 | @@ -179,7 +179,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface include conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, and mutation coverage for every signed receipt field. G2 remains `BLOCKED`: the reviewer disclosed a commercial conflict and did not cover the full required conformance scope. G4 also remains `BLOCKED`: one tagged external adapter candidate exists, but its clean-room provenance and behavior are not third-party validated and no separate validator result exists. +Version 0.4.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface include conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, mutation coverage for every signed receipt field, and independently authored adapter-level cases with target-instance tracing, complete outcomes, bounded proposition multiplicity, exact semantic mutations, and executable validator anti-vacuity controls. G2 remains `BLOCKED`: interested-party findings and internal remediation do not replace a complete independent review of the current surface. G4 also remains `BLOCKED`: one externally authored adapter candidate exists, but two independent implementations and a separately produced third-party validator result are not established. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/ROADMAP.md b/ROADMAP.md index 8943ca2..4a4b69b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -127,6 +127,12 @@ Only after the file-backed proof passes: 4. Define the complete adapter interface for enumeration, root-specific lineage-completeness evidence, quarantine and quarantine-phase coverage, store-owned repair inputs, retirement, reconstruction, recall probes, final dispositions, and coverage reporting. Empty enumeration or missing phase evidence is `unknown`; final success cannot overwrite a worse durable checkpoint. 5. Publish conformance vectors for signatures, sequencing, key rotation, concurrent events, invalid targets, receipt binding, and confidentiality. 6. Add model-assisted semantic probes behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. **Implemented internally:** [`prototype/semantic.py`](prototype/semantic.py), [`spec/semantic/`](spec/semantic/), and `errata semantic-test` record only structured, configuration-bound observations; inconclusive, malformed, missing, duplicate, or drifted required evidence is not success. +7. Publish adapter-level conformance with exact target-instance call controls, + complete outcomes, bounded proposition multiplicity, exact semantic + mutations, and executable validator anti-vacuity attacks. **Implemented + internally:** [`prototype/conformance.py`](prototype/conformance.py), + [`spec/adapter-conformance.json`](spec/adapter-conformance.json), and + `errata adapter-conformance`. Passing is internal evidence, not G2 or G4. The CLI quarantine checkpoint is also implemented internally. `errata quarantine` authenticates exactly the next pending erratum, gates enumerable descendants, diff --git a/SECURITY.md b/SECURITY.md index d010920..f34fdb9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,11 +4,15 @@ LLM Errata is currently a public concept proposal and may later include schemas, ## Supported versions -Until a later policy states otherwise, only the latest versioned release is eligible for security fixes. Development revisions after that release receive fixes at maintainer discretion and are not represented as supported releases. +Until a later policy states otherwise, only the latest versioned release is +eligible for security fixes. Support moves from 0.3.x to 0.4.x beginning with the immutable `v0.4.0` release. Development revisions after the latest release +receive fixes at maintainer discretion and are not represented as supported +releases. | Version | Supported | |---|---| -| 0.3.x | Yes | +| 0.4.x | Yes, after `v0.4.0` is published | +| 0.3.x | Yes, until `v0.4.0` is published | | 0.2.x and earlier | No | | Unreleased development revisions | No | diff --git a/SOURCES.md b/SOURCES.md index 5436c64..43bd6d7 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -49,6 +49,7 @@ the default branch and will move. | Source | Pin | Verified 2026-08-07 | |---|---|---| | [Inspeximus](https://github.com/DanceNitra/inspeximus) | `4c711f2982911841d86d7ac1989b0ffb866dc891` (review baseline, 2026-08-12); `v2.6.0` / `36611027a463a8e526e23baf2d6bb8d9797b67ac`; `v2.7.0` / `ccdb30de5a32896fafcb4ed18a7a6f228691e078` (2026-08-13) | Baseline confirms `retract_lineage`, `include_superseded`, `needs_rederivation`, and `rederive`. v2.6.0 makes known unresolved derivation holes `partially_audited` and reports `subject_reachable_records`. v2.7.0 adds a tagged LLM Errata adapter candidate and preserves explicit provenance disclosure: v2.6.1 copied reference structure and is disqualified; v2.7.0 claims a clean-room rewrite and reports adapter-contract defects reproduced locally here. Tags and source version are verified; GitHub Releases has no v2.6.0–v2.7.0 Release entries. The adapter targets historical LLM Errata commit `a477fe4f5c86730031b6285d9505778fb8eec060` and must rebind to the current immutable target. Candidate implementation and interested-party evidence only, not established independence, third-party validation, or G4 completion. | +| [DanceNitra adapter-conformance candidate](https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance) | `2ba1e299b3483b9038d03387345702427608b90b` (2026-08-13), MIT | Rastislav Drahos reported five adapter behaviors, three validator anti-vacuity attacks, and a duplicate-preservation counterexample. Inspeximus is a G4 candidate and the source commit discloses Claude Opus 5 co-authorship, so this is interested-party input rather than independent evidence. LLM Errata independently authored its corpus and validator without copying or vendoring the source runner or fixtures. | | [jedisct1/ed25519.py](https://github.com/jedisct1/ed25519.py) | `67902d339ea47418a60fb7684255b81bc4f6d46e` (2026-08-13) | Maintainer-provided pure-Python Ed25519 comparison implementation. README claims RFC 8032 vectors, canonical/small-order rejection, batch verification, and optional randomized signing countermeasures. Repository supplies no detected licence file or independent audit/build qualification, so it is a comparison oracle only and does not satisfy G3. | | [Doyle, A Truth Maintenance System](https://dblp.org/rec/journals/ai/Doyle79.html) | Artificial Intelligence 12(3), 1979; DBLP `journals/ai/Doyle79` | Stable bibliographic record confirms title and publication. Cited by Inspeximus as prior retract-and-retain lineage. Included to make historical dependency-retraction collision explicit; LLM Errata claims no invention of this mechanism. | | [IAB Data Deletion Request Framework](https://github.com/InteractiveAdvertisingBureau/Data-Subject-Rights/blob/main/Data%20Deletion%20Request%20Framework.md) | `b9418f5394ca91193181a61c567ffbbdff79cdef` (2025-02-06) | Repository unchanged since well before the cutoff | diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index da5b57d..3705a5b 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -29,6 +29,7 @@ mitigation or an admitted limit. | Erasure smuggling content back | An erasure carrying a replacement is refused | `errata._check_shape` | | Signature malleability | Non-canonical scalars (`S >= L`) rejected, so a receipt cannot be altered and still verify | `ed25519.verify` | | Repair that destroys retained memory | Preservation probe | `controller._run_triad` | +| Repair that duplicates retained propositions | Stable synthetic proposition identity/count; exact mutation control | `conformance.validate_adapter_conformance` | | Repair that adds the new value and keeps serving the old | Negative probe | `controller._run_triad` | | Half-repaired state being served | Quarantine completes before any rebuild; an interrupted repair leaves state gated | `controller.repair` | | Stale export undoing a repair | A re-import at or below the applied sequence is refused | `controller.reimport` | @@ -54,6 +55,12 @@ offers. Mitigations are outside the contract: owner-issued spot probes, an independent verifier, or a transparency log that makes a false claim durable and attributable. +**A lying conformance binding.** Target-instance tracing proves the controller +called the supplied adapter, and exact mutations prove the case can reject a +named flattering behavior. Neither proves the adapter truthfully exposed all +substrate state or proposition multiplicity. A third producer remains required +to validate independent implementations and the validator. + **A store whose substrate contradicts its API.** [Ghost Vectors](https://arxiv.org/abs/2606.18497v1) demonstrates that soft-deleted embeddings remain reconstructible from HNSW index files in diff --git a/VERSION b/VERSION index 0d91a54..1d0ba9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.4.0 diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 9e85ba8..753ab73 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -1,5 +1,23 @@ # Publication log +## 2026-08-13 — adapter-conformance feedback incorporated for 0.4.0 + +Rastislav Drahos/DanceNitra published an MIT-licensed candidate fixture at +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. Review reproduced +the reported 5/5 result and found false-pass paths in exception handling, +global name tracing, source binding, citation validation, declarative-only +anti-vacuity cases, and partial expected outcomes. LLM Errata independently +implemented the accepted behaviors with exact target-instance tracing, +complete outcomes, bounded proposition multiplicity, exact mutation results, +and executable validator attacks. The external runner and fixture were not +copied or vendored. + +This is internal remediation from interested-party feedback. It does not +satisfy G2 or G4. Repository ownership and licence terms are unchanged: +Thomas Rainer Willner retains copyright in repository-authored files; +independent specification implementations remain permitted under the +attributed implementation grant; Reference Code remains separately restricted. + This log records publication and outreach evidence. It does not upgrade a readiness gate. Last verified: 2026-08-13. diff --git a/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md b/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md new file mode 100644 index 0000000..1e510f9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md @@ -0,0 +1,231 @@ +# Adapter Conformance and Validator Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish an independently authored adapter-conformance corpus and fail-closed validator as LLM Errata 0.4.0 without upgrading blocked production-readiness gates. + +**Architecture:** A versioned JSON corpus describes complete adapter outcomes, normative predecessor citations, required target-instance calls, and exact mutation counter-results. `prototype/conformance.py` validates and executes the corpus through a provider-neutral binding protocol and reference binding; the CLI emits a canonical report. Release metadata records ownership, contributor provenance, and unchanged readiness status. + +**Tech Stack:** Python 3.11/3.13 standard library, JSON, unittest, existing LLM Errata controller/adapters, GitHub Actions and Releases. + +**Spec:** `docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md` + +## Global Constraints + +- LLM Errata files authored here remain owned by Thomas Willner. +- Do not copy or vendor `DanceNitra/agora` candidate runner or fixture. +- Preserve specific attribution to Rastislav Drahos/DanceNitra and immutable source commit. +- Preserve existing attributed commercial specification-implementation grant and restricted Reference Code boundary. +- Keep G2 through G6 blocked unless their existing independent evidence contracts pass. +- Standard-library only; all fixtures use synthetic data. +- Every production-code behavior starts with a failing public-seam test. +- The corpus pins the predecessor normative target `ac4468faf73c2cc7949dd29b2a2a151f5bd23116` and digest `7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12`. Runtime source identity is separately reported because a file cannot non-circularly contain the commit and digest that include its own bytes. + +--- + +### Task 1: Corpus and source/citation validation + +**Files:** +- Create: `spec/adapter-conformance.json` +- Create: `prototype/conformance.py` +- Create: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `load_corpus(path: Path, source_root: Path) -> AdapterCorpus` +- Produces: `ConformanceInputError` + +- [ ] **Step 1: Write failing corpus validation tests** + +Tests require exact schema keys, immutable commit availability, predecessor surface digest, exact source path/quotation, complete expectations, exact mutation counter-results, and provenance fields. Negative copies change one field at a time and must raise `ConformanceInputError`. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.CorpusValidation -v` +Expected: import failure because `prototype.conformance` does not exist. + +- [ ] **Step 3: Implement typed corpus parser and source binder** + +Use frozen dataclasses, exact-key validation, `git cat-file`/`git show`, and the pinned predecessor manifest/digest algorithm. Reject dirty identity ambiguity only when a run claims current committed identity; normative citation validation always reads immutable target bytes. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance.CorpusValidation -v` +Expected: all corpus validation tests pass. + +- [ ] **Step 5: Commit** + +`git commit -m "feat: define adapter conformance corpus"` + +### Task 2: Target-instance tracing and complete outcome comparison + +**Files:** +- Modify: `prototype/conformance.py` +- Modify: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `TracingAdapter` +- Produces: `compare_complete_outcome(expected, observed) -> tuple[str, ...]` +- Produces: `ConformanceBinding` protocol and `ReferenceConformanceBinding` + +- [ ] **Step 1: Write failing tracing and comparison tests** + +Tests prove an unrelated object with matching method names cannot satisfy the control, every unspecified triad/aggregate failure is rejected, and missing/extra observation fields fail. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.TargetTracing tests.test_conformance.CompleteComparison -v` +Expected: missing types/functions. + +- [ ] **Step 3: Implement proxy, binding protocol, reference binding, and strict comparator** + +Proxy only records calls routed through the exact wrapped instance. Reference binding builds synthetic lineage and exposes stable synthetic proposition labels/counts without private production data. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance.TargetTracing tests.test_conformance.CompleteComparison -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: trace exact adapter conformance outcomes"` + +### Task 3: Exact mutations and validator anti-vacuity controls + +**Files:** +- Modify: `prototype/conformance.py` +- Modify: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `validate_adapter_conformance(corpus_path, source_root, binding_factory) -> ConformanceReport` +- Produces: `run_validator_anti_vacuity_controls() -> tuple[ControlResult, ...]` + +- [ ] **Step 1: Write failing behavioral and mutation tests** + +Tests cover five honest cases, exact counter-results, exception-as-failure, mutation no-op rejection, empty receipt, no-op feed acceptance, and constant-`unknown` semantic aggregation. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.AdapterCases tests.test_conformance.AntiVacuity -v` + +- [ ] **Step 3: Implement execution, exact mutations, report, and controls** + +Each case runs honest and mutated bindings independently. Mutation success requires the declared complete semantic counter-result. Unexpected exceptions produce failed control evidence. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: reject vacuous conformance passes"` + +### Task 4: CLI and documentation + +**Files:** +- Modify: `prototype/cli.py` +- Modify: `tests/test_cli.py` +- Modify: `spec/README.md` +- Modify: `prototype/README.md` +- Modify: `IDEA.md` +- Modify: `ROADMAP.md` +- Modify: `THREAT_MODEL.md` +- Modify: `INDEPENDENT_IMPLEMENTATION.md` +- Modify: `SOURCES.md` + +**Interfaces:** +- Produces: `errata adapter-conformance --corpus ... --source-root ... --binding ...` +- Exit `0` pass, `1` behavioral/control failure, `2` invalid/inconclusive evidence. + +- [ ] **Step 1: Write failing subprocess CLI tests** + +Tests assert canonical JSON, source/runtime identity, five cases, three controls, provenance, evidence boundary, and all three exit classes. + +- [ ] **Step 2: Run CLI tests and verify RED** + +Run: `python3 -m unittest tests.test_cli.AdapterConformanceCommand -v` + +- [ ] **Step 3: Implement CLI and document normative multiplicity/provenance boundary** + +CLI imports `module:factory` only when explicitly supplied; default uses reference binding. Documentation states stable proposition identity/count scope and `unknown` behavior. + +- [ ] **Step 4: Run focused and full conformance tests** + +Run: `python3 -m unittest tests.test_conformance tests.test_cli -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: publish adapter conformance command"` + +### Task 5: Version 0.4.0 and release/readiness metadata + +**Files:** +- Modify: `VERSION` +- Modify: `CITATION.cff` +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `SECURITY.md` +- Modify: `readiness/production-readiness.json` +- Modify: `PRODUCTION_READINESS.md` +- Modify: `scripts/validate_repo.py` +- Modify: `tests/test_validate_repo.py` +- Modify: `scripts/check_readiness.py` +- Modify: `tests/test_readiness.py` +- Modify: `docs/PUBLICATION_LOG.md` + +**Interfaces:** +- Produces: internally aligned release version `0.4.0` +- Preserves: verdict `NOT_PROD_READY`, G1 `PASS`, G2-G6 `BLOCKED` + +- [ ] **Step 1: Write failing release-alignment tests** + +Tests require 0.4.0 across version, citation, maturity, security, ledger, matrix, changelog, and contributor provenance. They require unchanged gate statuses and current G2 evidence to name adapter conformance without treating it as external evidence. + +- [ ] **Step 2: Run release tests and verify RED** + +Run: `python3 -m unittest tests.test_validate_repo tests.test_readiness -v` + +- [ ] **Step 3: Update metadata and validators** + +Use release date 2026-08-13. Record independent implementation boundary and exact contributor provenance. Do not change licence grants or gate statuses. + +- [ ] **Step 4: Run release tests and verify GREEN** + +Run: `python3 -m unittest tests.test_validate_repo tests.test_readiness -v` + +- [ ] **Step 5: Commit** + +`git commit -m "release: prepare 0.4.0 adapter conformance hardening"` + +### Task 6: Full verification, review, integration, and publication + +**Files:** +- Review all changes since `c205fa07e818c682f2715e58c02b7e35bfe2ceb0` + +- [ ] **Step 1: Run full local gates** + +Run: `make check` +Run: `make publication` +Run: `make links` because `SOURCES.md` changes. + +- [ ] **Step 2: Run two-axis standards/spec review** + +Compare `c205fa0...HEAD` against AGENTS.md, CONTRIBUTING.md, and approved design. Fix all Critical/Important findings with focused regression tests. + +- [ ] **Step 3: Push scoped branch and update PR chain** + +Push `agent/adapter-conformance-release`, create PR against `agent/g2-publication`, and post exact test/provenance/readiness evidence. + +- [ ] **Step 4: Obtain exact-head CI and integrate in dependency order** + +Require Python 3.11 and 3.13 checks on exact head. Merge adapter branch, PR #8, then PR #3 only when each resulting head is verified and no new actionable feedback exists. + +- [ ] **Step 5: Publish release** + +Backfill GitHub release object for immutable `v0.3.0` if absent. Tag current verified `main` as `v0.4.0`, publish release notes with `NOT_PROD_READY`, ownership/licence boundary, contributor thanks, and G2-G6 blockers. + +- [ ] **Step 6: Reply to contributors** + +Freshly inventory all external authors. Reply individually with what their contribution changed, exact release evidence, unchanged gate limits, and one optional focused follow-up where useful. + +- [ ] **Step 7: Final freshness sweep** + +Repeat all GitHub surfaces, classify every delta, validate schema-v2 freshness receipt, and reopen loop for any new actionable feedback. diff --git a/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md new file mode 100644 index 0000000..18a45cd --- /dev/null +++ b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md @@ -0,0 +1,236 @@ +# Adapter Conformance and Validator Hardening Design + +**Date:** 2026-08-13 +**Owner:** Thomas Willner +**Status:** Approved for implementation +**Target release:** 0.4.0 + +## Purpose + +LLM Errata currently exercises its wire contract, controller, semantic +aggregator, and receipt binding more deeply than it exercises independently +authored store adapters. Rastislav Drahos reported that an external runner +could reproduce 28 of 28 published cases while only one case reached any store +adapter. He then published candidate adapter cases and validator anti-vacuity +requirements at immutable commit +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. + +This change adopts the verified behavioral findings without copying or +vendoring the contributed runner. It adds an independently authored, +provider-neutral adapter-conformance corpus and a fail-closed validator whose +own controls must demonstrably fail against flattering implementations. + +Release 0.4.0 remains an experimental conformance proposal and tested +reference implementation. It does not change the project verdict from +`NOT_PROD_READY`, and it does not satisfy G2 or G4. + +## Ownership, licence, and provenance + +- LLM Errata, its repository, specification, project name, and files authored + in this repository remain owned and governed by Thomas Willner. +- Repository publication and versioned releases do not transfer copyright. +- Rastislav Drahos and DanceNitra retain copyright in their candidate fixture. + Its MIT licence permits reuse but does not assign ownership. +- This repository will not copy or vendor that runner or its fixture files. + Implementation is independently authored from the accepted behavioral + requirements and the LLM Errata normative contract. +- Documentation must credit Rastislav Drahos and DanceNitra for the reported + adapter-coverage gap, duplicate-preservation counterexample, candidate cases, + and anti-vacuity findings. It must identify the immutable source commit, + MIT licence, interested-party conflict, and disclosed Claude Opus 5 + co-authorship. +- Existing LLM Errata licence terms remain unchanged. Independent commercial + and non-commercial implementations of the Specification Materials are + already permitted by the repository's irrevocable, royalty-free + implementation grant, subject to accessible credit to LLM Errata and Thomas + Willner and the no-false-endorsement condition. Commercial use, copying, + redistribution, hosting, or adaptation of Reference Code still requires + separate written permission unless another licence expressly applies. + Technical adoption, contributor credit, or release publication does not + imply endorsement, partnership, certification, or transfer of copyright. + +## Normative preservation rule + +Preservation is strengthened from boolean recallability to bounded +multiplicity preservation. + +Within a declared, inspectable adapter scope, repair must not increase the +active multiplicity of a preserved proposition unless the erratum explicitly +requires an additional assertion. A proposition is identified by the +adapter-provided stable proposition identity used by its conformance +observation interface, not by lossy text normalization or embedding +similarity. + +The conformance observation for each active proposition contains: + +- a stable, provider-local proposition identifier; +- a content-free or synthetic fixture label suitable for comparison; and +- an active assertion count. + +Adapters unable to expose proposition identity or active multiplicity cannot +pass the cardinality case. They report `unknown` for that observation. This +rule is scoped evidence: it does not claim semantic uniqueness outside the +declared adapter surface. + +## Corpus format + +`spec/adapter-conformance.json` is a versioned, provider-neutral corpus. Each +case contains: + +- stable case ID and operation; +- synthetic initial state; +- complete expected checkpoint, aggregate, triad, receipt, and store + observations; +- required adapter method calls; +- a pinned normative source path and exact quotation; +- one named mutation and the complete result that mutation must produce; and +- provenance and evidence-boundary fields. + +Partial expected outcomes are forbidden. A case fails when any extra triad +failure, worse aggregate result, missing store observation, unexpected +exception, or unspecified result appears. + +Initial behavioral cases cover: + +1. undeclared derivatives cannot become `verified`; +2. complete lineage can reach `verified`; +3. a rebuild cannot increase preserved-proposition multiplicity; +4. erasure evidence is content-free and non-vacuous; and +5. collateral survives supersession while the complete repair still passes. + +## Validator architecture + +`prototype/conformance.py` owns parsing, source binding, execution, and +result comparison. It exposes one public entry point: + +`validate_adapter_conformance(corpus_path, source_root, binding_factory) -> ConformanceReport`. + +The validator is standard-library only. + +### Source identity + +The corpus binds: + +- exact LLM Errata Git commit; +- canonical conformance-surface digest; +- fixture schema version; and +- every normative source path and exact quotation. + +Validation refuses a dirty or different source tree, a commit mismatch, a +digest mismatch, a missing source, or quotation drift. Source mismatch is an +invalid run, never a case failure or pass. + +### Target-instance tracing + +Required calls are observed through a proxy around the exact adapter instance +given to the controller. The proxy records protocol member access and calls. +Global frame names, unrelated modules, helper functions, and another object +with the same method name cannot satisfy a positive control. + +### Mutation controls + +Every case runs twice: + +1. honest binding; +2. the declared flattering mutation installed through the public adapter or + validator seam. + +The mutation must complete and produce the exact declared counter-result. +Any unexpected exception, missing result, different failure, or unchanged pass +fails the control. Exceptions count only when the corpus explicitly declares +that exact exception type and message as the required outcome. + +### Validator anti-vacuity controls + +Three executable meta-tests attack the validator: + +- an empty receipt cannot satisfy confidentiality; +- no-op feed verification cannot satisfy accept-side feed cases; and +- a constant-`unknown` aggregator cannot satisfy the semantic corpus. + +These are validator requirements, not adapter scores. A release cannot claim +validator readiness unless all three mutations are rejected for their +specified semantic reason. + +## Reference binding + +The LLM Errata reference binding uses only public adapter/controller +interfaces plus a conformance observation seam. Production adapters are not +required to expose raw private store contents. A binding may expose only +synthetic fixture identities and counts for an isolated conformance run. + +The corpus and validator must run against the repository reference adapter. +Passing the reference binding demonstrates harness behavior, not independent +interoperability. + +## CLI and evidence + +The existing CLI gains an `adapter-conformance` command accepting: + +- corpus path; +- source root; and +- an importable binding factory. + +Output is one canonical JSON report with source identity, per-case results, +positive controls, mutation controls, anti-vacuity results, provenance, and an +explicit statement that the report is not G2 or G4 evidence. + +Exit codes: + +- `0`: corpus valid and every case/control passes; +- `1`: one or more behavioral or mutation controls fail; +- `2`: source, corpus, binding, or execution evidence is invalid or + inconclusive. + +## Release integration + +Version 0.4.0 includes the current `agent/g2-publication` surface plus this +validator hardening. The release: + +- updates `VERSION`, `CITATION.cff`, README maturity text, supported + security version, readiness ledger/matrix, and changelog; +- records contributor provenance and the independently authored implementation + boundary; +- keeps every G2 through G6 status unchanged unless its existing evidence + contract independently passes; +- backfills the missing GitHub release object for immutable tag `v0.3.0` + without moving that tag; and +- creates immutable tag and GitHub release `v0.4.0` only after exact-head + local checks and GitHub CI pass. + +## Verification seams + +TDD is performed at these public seams: + +1. corpus validation through `validate_adapter_conformance`; +2. target adapter calls through the tracing proxy; +3. exact mutation-result comparison; +4. executable validator anti-vacuity suite; +5. CLI canonical report and exit codes; +6. repository version/readiness/publication validation; and +7. release metadata consistency. + +Required final checks: + +- focused conformance and CLI tests; +- `make check`; +- `make publication`; +- `make links` when source citations change; +- two-axis standards/spec review against this design; +- exact-head CI on Python 3.11 and 3.13; +- immutable tag and GitHub release verification; and +- repository-wide GitHub freshness receipt after the last mutation. + +## Safe stop + +Stop without releasing when any of these occurs: + +- the normative cardinality rule cannot be expressed without provider-specific + private data; +- the validator cannot distinguish its own flattering mutations; +- source identity cannot be bound mechanically; +- current branch integration drops or rewrites existing evidence; +- local or exact-head CI fails; +- a new actionable contributor comment or review changes the required design; + or +- version, tag, release, licence, attribution, or readiness metadata disagree. diff --git a/prototype/README.md b/prototype/README.md index f84e8ef..b206869 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -136,6 +136,19 @@ inputs ephemerally, but only the structured verdict, binding digest, timestamp, and response digest may enter an observation. Raw output and erased values are not persisted. +### Adapter-conformance command + +`adapter-conformance` runs the checked-in provider-neutral adapter corpus +without creating a workspace. The default reference binding drives the real +`Importer` lifecycle through a proxy around the exact adapter instance. A +third-party binding is supplied as `--binding module:factory`. + +The canonical JSON report separates the immutable normative predecessor target +from the runtime commit being exercised, lists complete honest and mutation +outcomes, records exact target-instance calls, and includes three executable +validator attacks. Passing remains internal reference evidence and does not +upgrade G2 or G4. + Running it against a real SQLite store produces the result that matters: ```text diff --git a/prototype/cli.py b/prototype/cli.py index 1816bb1..ff0a596 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -28,10 +28,16 @@ from prototype.adapters import Coverage, OpaqueAdapter from prototype.checkpoints import CheckpointError +from prototype.conformance import ( + ConformanceInputError, + ReferenceConformanceBinding, + load_binding_factory, + validate_adapter_conformance, +) from prototype.controller import Importer, Phase from prototype.errata import Erratum, FeedError, Operation, RootRegistry, read_feed from prototype.lineage import LineageLedger -from prototype.receipts import Receipt +from prototype.receipts import Receipt, receipt_acceptance_errors from prototype.schema import load as load_schema, validate as validate_schema from prototype.semantic import ( RecordedSemanticVerifier, @@ -43,7 +49,7 @@ ) from prototype.signing import Ed25519Signer from prototype.sqlite_store import SqliteAdapter -from prototype.workspace import Workspace +from prototype.workspace import ReceiptReadError, Workspace EXIT_OK = 0 @@ -51,6 +57,39 @@ EXIT_INCONCLUSIVE = 2 +def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: + """Run provider-neutral adapter cases and validator self-controls.""" + + try: + if args.binding is None: + factory = ReferenceConformanceBinding + source = None + binding_root = args.binding_root + else: + binding_root = args.binding_root or args.source_root + with load_binding_factory(args.binding, binding_root) as (factory, source): + report = validate_adapter_conformance( + args.corpus, + args.source_root, + factory, + binding_root=binding_root, + binding_source=source, + ) + if args.binding is None: + report = validate_adapter_conformance( + args.corpus, + args.source_root, + factory, + binding_root=binding_root, + binding_source=source, + ) + except ConformanceInputError as error: + print(f"invalid conformance evidence: {error}", file=sys.stderr) + return EXIT_INCONCLUSIVE + print(report.canonical_json()) + return EXIT_OK if report.passed else EXIT_REFUSED + + def _load_json(path: Path, *, label: str) -> object: try: return json.loads(path.read_text(encoding="utf-8")) @@ -317,16 +356,23 @@ def cmd_audit(ws: Workspace, args: argparse.Namespace) -> int: def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: """Check every receipt against the published key and the published schema.""" - receipts = ws.all_receipts() + try: + receipts = ws.all_receipts() + except ReceiptReadError as error: + print(f" {error} -> BAD") + return EXIT_REFUSED if not receipts: print("no receipts to verify", file=sys.stderr) return EXIT_REFUSED key = ws.importer_verification_key() - schema = load_schema("receipt") bad = 0 for name, payload in receipts: - errors = validate_schema(payload, schema) + errors = receipt_acceptance_errors(payload) + if errors: + bad += 1 + print(f" {name}: signature=not-checked schema={errors[0]} -> BAD") + continue signature = payload.get("signature") rebuilt = Receipt( importer=payload["importer"], @@ -368,6 +414,7 @@ def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: "audit": cmd_audit, "verify": cmd_verify, "semantic-test": cmd_semantic_test, + "adapter-conformance": cmd_adapter_conformance, } @@ -426,13 +473,22 @@ def build_parser() -> argparse.ArgumentParser: semantic_test.add_argument("--config", required=True, type=Path) semantic_test.add_argument("--observations", required=True, type=Path) semantic_test.add_argument("--case", default="verified-correction") + + adapter_conformance = sub.add_parser( + "adapter-conformance", + help="run adapter cases and validator anti-vacuity controls", + ) + adapter_conformance.add_argument("--corpus", required=True, type=Path) + adapter_conformance.add_argument("--source-root", required=True, type=Path) + adapter_conformance.add_argument("--binding-root", type=Path) + adapter_conformance.add_argument("--binding") return parser def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) ws = Workspace(args.workspace) - if args.command not in {"init", "semantic-test"} and not ws.exists(): + if args.command not in {"init", "semantic-test", "adapter-conformance"} and not ws.exists(): print( f"no workspace at {args.workspace}; run `errata init` first", file=sys.stderr, diff --git a/prototype/conformance.py b/prototype/conformance.py new file mode 100644 index 0000000..4d03190 --- /dev/null +++ b/prototype/conformance.py @@ -0,0 +1,1200 @@ +"""Provider-neutral adapter conformance with fail-closed self-controls.""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.abc +import importlib.util +import inspect +import json +import re +import signal +import subprocess +import sys +import threading +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + + +class ConformanceInputError(ValueError): + """Corpus or source evidence cannot support a conformance run.""" + + +class TracingAdapter: + """Proxy that records calls made through one exact adapter instance.""" + + def __init__(self, target: object) -> None: + object.__setattr__(self, "target", target) + object.__setattr__(self, "_calls", []) + + @property + def calls(self) -> tuple[str, ...]: + return tuple(object.__getattribute__(self, "_calls")) + + def reset_calls(self) -> None: + """Start a new trace window around controller-issued lifecycle calls.""" + + object.__getattribute__(self, "_calls").clear() + + def __getattr__(self, name: str) -> Any: + value = getattr(self.target, name) + if not callable(value): + return value + + def traced(*args: Any, **kwargs: Any) -> Any: + object.__getattribute__(self, "_calls").append(name) + return value(*args, **kwargs) + + return traced + + +def compare_complete_outcome( + expected: dict[str, Any], observed: dict[str, Any] +) -> tuple[str, ...]: + """Return every exact structural/value difference between two outcomes.""" + + failures: list[str] = [] + + def compare(want: object, got: object, path: str) -> None: + if isinstance(want, dict): + if not isinstance(got, dict): + failures.append(f"{path}: expected object, got {type(got).__name__}") + return + for key in want: + child = f"{path}.{key}" if path else key + if key not in got: + failures.append(f"{child}: missing") + else: + compare(want[key], got[key], child) + for key in got: + if key not in want: + child = f"{path}.{key}" if path else key + failures.append(f"{child}: unexpected") + return + if type(want) is not type(got) or want != got: + failures.append(f"{path}: expected {want!r}, got {got!r}") + + compare(expected, observed, "") + return tuple(failures) + + +@dataclass(frozen=True) +class NormativeTarget: + commit: str + surface_digest: str + + +@dataclass(frozen=True, order=True) +class PropositionObservation: + """Content-safe multiplicity for one stable provider-local proposition.""" + + proposition_id: str + fixture_label: str + active_count: int + + def __post_init__(self) -> None: + if not self.proposition_id or not self.fixture_label: + raise ConformanceInputError("proposition identity and label must be non-empty") + if isinstance(self.active_count, bool) or self.active_count < 0: + raise ConformanceInputError("proposition active count must be non-negative") + + +def compare_proposition_multiplicity( + before: tuple[PropositionObservation, ...] | None, + after: tuple[PropositionObservation, ...] | None, +) -> str: + """Compare exact stable identities; text similarity is never an identity seam.""" + + if before is None or after is None: + return "unknown" + before_by_id = {item.proposition_id: item for item in before} + after_by_id = {item.proposition_id: item for item in after} + if len(before_by_id) != len(before) or len(after_by_id) != len(after): + return "unknown" + for proposition_id, prior in before_by_id.items(): + current = after_by_id.get(proposition_id) + if current is None or current.fixture_label != prior.fixture_label: + return "unknown" + if current.active_count > prior.active_count: + return "increased" + return "known" + + +@dataclass(frozen=True) +class AdapterCase: + value: dict[str, Any] + + @property + def case_id(self) -> str: + return self.value["id"] + + +@dataclass(frozen=True) +class AdapterCorpus: + schema_version: int + normative_target: NormativeTarget + provenance: dict[str, str] + cases: tuple[AdapterCase, ...] + validator_controls: tuple[dict[str, str], ...] + status: str + evidence_boundary: str + sha256: str + + +@dataclass(frozen=True) +class CaseResult: + case_id: str + expectation_met: bool + failures: tuple[str, ...] + required_calls: tuple[str, ...] + observed_calls: tuple[str, ...] + missing_calls: tuple[str, ...] + positive_control_passed: bool + mutation_control_passed: bool + mutation_failures: tuple[str, ...] + observed: dict[str, Any] + mutation_observed: dict[str, Any] | None + + @property + def passed(self) -> bool: + return ( + self.expectation_met + and self.positive_control_passed + and self.mutation_control_passed + ) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.case_id, + "passed": self.passed, + "expectation_met": self.expectation_met, + "failures": list(self.failures), + "required_calls": list(self.required_calls), + "observed_calls": list(self.observed_calls), + "missing_calls": list(self.missing_calls), + "positive_control_passed": self.positive_control_passed, + "mutation_control_passed": self.mutation_control_passed, + "mutation_failures": list(self.mutation_failures), + "observed": self.observed, + "mutation_observed": self.mutation_observed, + } + + +@dataclass(frozen=True) +class ControlResult: + control_id: str + passed: bool + observed_failure: str + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.control_id, + "passed": self.passed, + "observed_failure": self.observed_failure, + } + + +@dataclass(frozen=True) +class ConformanceReport: + binding: str + normative_commit: str + normative_surface_digest: str + corpus_sha256: str + runtime_commit: str | None + runtime_tree: str + binding_source: dict[str, str] + cases: tuple[CaseResult, ...] + validator_controls: tuple[ControlResult, ...] + provenance: dict[str, str] + evidence_boundary: str + + @property + def passed(self) -> bool: + return all(item.passed for item in self.cases) and all( + item.passed for item in self.validator_controls + ) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": 1, + "binding": self.binding, + "passed": self.passed, + "normative_target": { + "commit": self.normative_commit, + "surface_digest": self.normative_surface_digest, + }, + "corpus_sha256": self.corpus_sha256, + "runtime_commit": self.runtime_commit, + "runtime_tree": self.runtime_tree, + "binding_source": dict(sorted(self.binding_source.items())), + "cases": [item.to_dict() for item in self.cases], + "validator_controls": [ + item.to_dict() for item in self.validator_controls + ], + "provenance": dict(sorted(self.provenance.items())), + "evidence_boundary": self.evidence_boundary, + } + + def canonical_json(self) -> str: + return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + + +ROOT_KEYS = { + "schema_version", "status", "evidence_boundary", "normative_target", + "provenance", "cases", "validator_controls", +} +TARGET_KEYS = {"commit", "surface_digest"} +PROVENANCE_KEYS = { + "reported_by", "source_url", "source_commit", "source_license", + "relationship", "ai_assistance", "implementation", +} +CASE_KEYS = { + "id", "operation", "normative", "scenario", "required_calls", + "expected", "mutation", +} +NORMATIVE_KEYS = {"commit", "path", "quote"} +EXPECTED_KEYS = {"checkpoint", "aggregate", "triad", "store", "receipt"} +STORE_KEYS = {"multiplicity", "erased_absent", "preserved_present", "unrelated_present"} +RECEIPT_KEYS = {"names_store", "non_trivial", "forbidden_absent"} +MUTATION_KEYS = {"id", "exact_counter_result"} +CONTROL_KEYS = {"id", "mutation", "required_failure"} +REQUIRED_TARGET = "ac4468faf73c2cc7949dd29b2a2a151f5bd23116" +REQUIRED_DIGEST = "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" +GIT_TIMEOUT_SECONDS = 10.0 +BINDING_TIMEOUT_SECONDS = 10.0 +REQUIRED_PROVENANCE = { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored.", +} +REQUIRED_CONTROLS = ( + { + "id": "empty-receipt-must-fail", + "mutation": "empty-receipt", + "required_failure": "receipt is vacuous", + }, + { + "id": "no-op-feed-verification-must-fail", + "mutation": "no-op-feed-verification", + "required_failure": "accepted feed is incomplete", + }, + { + "id": "constant-unknown-aggregator-must-fail", + "mutation": "constant-unknown-aggregator", + "required_failure": "semantic verdict diversity is missing", + }, +) + + +def _exact(value: object, keys: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != keys: + raise ConformanceInputError(f"{label} must contain exactly {sorted(keys)}") + return value + + +def _nonempty(value: object, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ConformanceInputError(f"{label} must be non-empty") + return value + + +def _git(root: Path, *args: str) -> bytes: + try: + result = subprocess.run( + ["git", *args], cwd=root, capture_output=True, check=False, + timeout=GIT_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as error: + raise ConformanceInputError("Git source verification timed out") from error + if result.returncode != 0: + raise ConformanceInputError("immutable source commit is unavailable") + return result.stdout + + +def _surface_paths_at_commit(root: Path, commit: str) -> tuple[str, ...]: + required_tests = ( + "tests/test_adapters.py", "tests/test_checkpoints.py", "tests/test_cli.py", + "tests/test_controller.py", "tests/test_ed25519.py", + "tests/test_errata_feed.py", "tests/test_schema.py", + "tests/test_semantic.py", "tests/test_sqlite_store.py", + ) + listed = _git(root, "ls-tree", "-r", "--name-only", commit).decode("utf-8").splitlines() + files = set(listed) + groups = ( + tuple(sorted(path for path in files if re.fullmatch(r"prototype/[^/]+\.py", path))), + ("prototype/README.md", "spec/README.md"), + tuple(sorted(path for path in files if re.fullmatch(r"spec/[^/]+\.schema\.json", path))), + tuple(sorted(path for path in files if re.fullmatch(r"spec/vectors/[^/]+\.json", path))), + tuple(sorted(path for path in files if re.fullmatch(r"spec/semantic/[^/]+\.json", path))), + ("ROADMAP.md", "THREAT_MODEL.md", "SECURITY.md"), + required_tests, + ) + paths = tuple(sorted(item for group in groups for item in group)) + if any(path not in files for path in paths): + raise ConformanceInputError("canonical surface is incomplete") + return paths + + +def _surface_digest_at_commit(root: Path, commit: str) -> str: + digest = hashlib.sha256() + for relative in _surface_paths_at_commit(root, commit): + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(_git(root, "show", f"{commit}:{relative}")) + digest.update(b"\0") + return digest.hexdigest() + + +def _validate_outcome(value: object, operation: str, label: str) -> dict[str, Any]: + outcome = _exact(value, EXPECTED_KEYS, f"{label} expected outcome") + triad_keys = {"negative", "preserve"} if operation == "erase" else { + "negative", "positive", "preserve" + } + _exact(outcome["triad"], triad_keys, f"{label} expected outcome triad") + _exact(outcome["store"], STORE_KEYS, f"{label} expected outcome store") + _exact(outcome["receipt"], RECEIPT_KEYS, f"{label} expected outcome receipt") + if outcome["checkpoint"] not in {"verified", "partial", "unknown", "failed"}: + raise ConformanceInputError(f"{label} expected outcome checkpoint is invalid") + if outcome["aggregate"] not in {"verified", "partial", "unknown", "failed"}: + raise ConformanceInputError(f"{label} expected outcome aggregate is invalid") + if set(outcome["triad"].values()) - {"pass", "fail"}: + raise ConformanceInputError(f"{label} expected outcome triad is invalid") + return outcome + + +def load_corpus( + path: Path, source_root: Path, *, require_canonical_path: bool = False +) -> AdapterCorpus: + if require_canonical_path and path.resolve() != ( + source_root / "spec" / "adapter-conformance.json" + ).resolve(): + raise ConformanceInputError("corpus is not the canonical checked-in corpus") + try: + corpus_bytes = path.read_bytes() + payload = json.loads(corpus_bytes.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ConformanceInputError("corpus is not readable canonical JSON") from error + root = _exact(payload, ROOT_KEYS, "corpus") + if root["schema_version"] != 1: + raise ConformanceInputError("corpus schema version must be 1") + target = _exact(root["normative_target"], TARGET_KEYS, "normative target") + if target["commit"] != REQUIRED_TARGET: + raise ConformanceInputError("normative target commit is not canonical") + if target["surface_digest"] != REQUIRED_DIGEST: + raise ConformanceInputError("normative surface digest is not canonical") + actual_digest = _surface_digest_at_commit(source_root, target["commit"]) + if actual_digest != target["surface_digest"]: + raise ConformanceInputError("normative surface digest does not match source") + + provenance = _exact(root["provenance"], PROVENANCE_KEYS, "provenance") + if provenance != REQUIRED_PROVENANCE: + raise ConformanceInputError("provenance does not match the accepted contribution") + + raw_cases = root["cases"] + if not isinstance(raw_cases, list) or len(raw_cases) != 5: + raise ConformanceInputError("corpus must contain exactly five cases") + cases = [] + ids = set() + for index, value in enumerate(raw_cases): + case = _exact(value, CASE_KEYS, f"case[{index}]") + case_id = _nonempty(case["id"], f"case[{index}] id") + if case_id in ids: + raise ConformanceInputError("case IDs must be unique") + ids.add(case_id) + if case["operation"] not in {"correct", "supersede", "erase"}: + raise ConformanceInputError(f"{case_id} operation is invalid") + normative = _exact(case["normative"], NORMATIVE_KEYS, f"{case_id} normative") + commit = _nonempty(normative["commit"], f"{case_id} normative commit") + try: + _git(source_root, "merge-base", "--is-ancestor", commit, "HEAD") + except ConformanceInputError as error: + raise ConformanceInputError( + f"{case_id} normative commit is not reachable from runtime history" + ) from error + relative = _nonempty(normative["path"], f"{case_id} normative path") + quote = _nonempty(normative["quote"], f"{case_id} normative quotation") + source = _git(source_root, "show", f"{commit}:{relative}").decode("utf-8") + if quote not in source: + raise ConformanceInputError(f"{case_id} normative quotation does not match source") + calls = case["required_calls"] + if not isinstance(calls, list) or not calls or not all( + isinstance(item, str) and item for item in calls + ) or len(calls) != len(set(calls)): + raise ConformanceInputError(f"{case_id} required calls are invalid") + _validate_outcome(case["expected"], case["operation"], case_id) + mutation = _exact(case["mutation"], MUTATION_KEYS, f"{case_id} mutation") + _nonempty(mutation["id"], f"{case_id} mutation id") + _validate_outcome( + mutation["exact_counter_result"], case["operation"], f"{case_id} mutation" + ) + cases.append(AdapterCase(case)) + + raw_controls = root["validator_controls"] + if not isinstance(raw_controls, list) or len(raw_controls) != 3: + raise ConformanceInputError("corpus must contain exactly three validator controls") + controls = [] + for index, value in enumerate(raw_controls): + control = _exact(value, CONTROL_KEYS, f"validator control[{index}]") + if any(not isinstance(item, str) or not item.strip() for item in control.values()): + raise ConformanceInputError("validator control fields must be non-empty") + controls.append(control) + if tuple(controls) != REQUIRED_CONTROLS: + raise ConformanceInputError("validator controls do not match the executable attacks") + return AdapterCorpus( + schema_version=1, + normative_target=NormativeTarget(target["commit"], target["surface_digest"]), + provenance=dict(provenance), + cases=tuple(cases), + validator_controls=tuple(controls), + status=_nonempty(root["status"], "status"), + evidence_boundary=_nonempty(root["evidence_boundary"], "evidence boundary"), + sha256=hashlib.sha256(corpus_bytes).hexdigest(), + ) + + +class _SyntheticHit: + def __init__(self, artifact_id: str, content: str) -> None: + self.artifact_id = artifact_id + self.content = content + + +class ReferenceConformanceAdapter: + """Synthetic store exercising the public StoreAdapter contract.""" + + name = "reference-conformance" + required = True + + def __init__(self, *, undeclared: bool = False) -> None: + self._records = { + "diet": { + "text": "is vegetarian", "inputs": (), "root": "fact:diet", + "propositions": ("fixture:diet",), + }, + "quiet": { + "text": "prefers quiet restaurants", "inputs": (), "root": "fact:quiet", + "propositions": ("fixture:quiet",), + }, + "budget": { + "text": "moderate budget", "inputs": (), "root": "fact:budget", + "propositions": ("fixture:budget",), + }, + "pet": { + "text": "has a cat", "inputs": (), "root": "fact:pet", + "propositions": ("fixture:pet",), + }, + "summary": { + "text": "is vegetarian; prefers quiet restaurants; moderate budget", + "inputs": ("diet", "quiet", "budget"), "root": None, + "propositions": ( + "fixture:diet", "fixture:quiet", "fixture:budget", + ), + }, + } + if undeclared: + self._records["orphan"] = { + "text": "synthetic undeclared derivative", + "inputs": (), + "root": None, + "propositions": (), + } + self._active = {key: value["text"] for key, value in self._records.items()} + self._active_propositions = { + key: tuple(value["propositions"]) for key, value in self._records.items() + } + self._quarantined: set[str] = set() + self._retired: set[str] = set() + self._rebuilt: set[str] = set() + self._undeclared = undeclared + self._force_lineage_complete = False + self._force_unknown = False + self._duplicate_inputs = False + self._empty_receipt = False + self._retire_all = False + + def enumerate(self, root: str) -> tuple[str, ...]: + if root != "fact:diet": + return () + return ("diet", "summary") + + def lineage_complete(self, root: str) -> bool: + return self._force_lineage_complete or not self._undeclared + + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: + self._quarantined.update(artifact_ids) + + def is_quarantined(self, artifact_id: str) -> bool: + return artifact_id in self._quarantined + + def quarantine_coverage(self, root: str): + from prototype.adapters import Coverage + + if self._force_unknown or not self.lineage_complete(root): + return Coverage.UNKNOWN + descendants = set(self.enumerate(root)) + return Coverage.VERIFIED if descendants <= self._quarantined else Coverage.FAILED + + def source_artifact(self, artifact_id: str) -> str: + return artifact_id + + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + return tuple(self._records[artifact_id]["inputs"]) + + def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: + targets = tuple(self._active) if self._retire_all else (artifact_id,) + for target in targets: + self._retired.add(target) + self._active.pop(target, None) + self._active_propositions.pop(target, None) + + def rebuild( + self, artifact_id: str, *, inputs: tuple[str, ...], replacement: str | None + ) -> str: + if self._retire_all: + return "" + parts = [self._records[item]["text"] for item in inputs] + if replacement: + parts.insert(0, replacement) + text = "; ".join(parts) + self._active[artifact_id] = text + propositions = tuple( + proposition + for item in inputs + for proposition in self._records[item]["propositions"] + ) + self._active_propositions[artifact_id] = propositions + if self._duplicate_inputs: + for item in inputs: + duplicate = f"duplicate:{item}" + self._active[duplicate] = self._records[item]["text"] + self._active_propositions[duplicate] = tuple( + self._records[item]["propositions"] + ) + self._rebuilt.add(artifact_id) + self._quarantined.discard(artifact_id) + return text + + def recall(self, query: str) -> tuple[_SyntheticHit, ...]: + term = query.lower() + return tuple( + _SyntheticHit(key, text) + for key, text in self._active.items() + if key not in self._quarantined and term in text.lower() + ) + + def snapshot(self) -> dict[str, str]: + return dict(self._active) + + def coverage(self, root: str): + from prototype.adapters import Coverage + + if self._force_unknown or not self.lineage_complete(root): + return Coverage.UNKNOWN + descendants = set(self.enumerate(root)) + disposed = self._retired | self._rebuilt + return Coverage.VERIFIED if descendants <= disposed else Coverage.FAILED + + def dispositions(self, root: str) -> dict[str, str]: + result = {} + for item in self.enumerate(root): + if item in self._retired: + result[item] = "retired" + elif item in self._rebuilt: + result[item] = "rebuilt" + elif item in self._quarantined: + result[item] = "quarantined-only" + else: + result[item] = "untouched" + return result + + def proposition_observations(self) -> tuple[PropositionObservation, ...]: + labels = { + "fixture:diet": "diet", + "fixture:quiet": "quiet", + "fixture:budget": "budget", + "fixture:pet": "pet", + } + counts = {proposition_id: 0 for proposition_id in labels} + for identities in self._active_propositions.values(): + for proposition_id in identities: + counts[proposition_id] = counts.get(proposition_id, 0) + 1 + return tuple( + PropositionObservation(proposition_id, labels[proposition_id], count) + for proposition_id, count in sorted(counts.items()) + ) + + +class ReferenceConformanceBinding: + """Reference binding; external implementations supply an equivalent class.""" + + name = "llm-errata-reference" + preserved_proposition_ids = frozenset( + {"fixture:quiet", "fixture:budget", "fixture:pet"} + ) + + @classmethod + def preserved_observations( + cls, adapter: ReferenceConformanceAdapter + ) -> tuple[PropositionObservation, ...]: + return tuple( + item + for item in adapter.proposition_observations() + if item.proposition_id in cls.preserved_proposition_ids + ) + + def build(self, case: AdapterCase): + from prototype.controller import Importer + from prototype.errata import RootRegistry + from prototype.lineage import LineageLedger + from prototype.signing import Ed25519Signer + + adapter = ReferenceConformanceAdapter( + undeclared=case.value["scenario"] == "undeclared-derivative" + ) + traced = TracingAdapter(adapter) + owner = Ed25519Signer(b"conformance-owner") + importer = Importer( + "conformance-importer", + ledger=LineageLedger(), + adapters=[traced], + signer=Ed25519Signer(b"conformance-importer"), + owner=owner.public, + roots=RootRegistry({"fact:diet"}), + ) + return importer, traced, { + "adapter": adapter, + "owner": owner, + "before_observations": self.preserved_observations(adapter), + } + + def erratum(self, case: AdapterCase, context: dict[str, Any]): + from prototype.errata import Erratum, Operation + + operation = Operation(case.value["operation"]) + postconditions = { + "negative": "vegetarian", + "preserve": "quiet restaurants|moderate budget|cat", + } + replacement = None + if operation is not Operation.ERASE: + replacement = "eats meat again" + postconditions["positive"] = replacement + return context["owner"].sign_erratum( + Erratum( + erratum_id=f"case-{case.case_id}", + sequence=1, + target_root="fact:diet", + operation=operation, + valid_from="2026-08-01T00:00:00Z", + replacement=replacement, + postconditions=postconditions, + ) + ) + + def apply_mutation( + self, case: AdapterCase, importer: object, adapter: TracingAdapter, + context: dict[str, Any], + ) -> None: + target = context["adapter"] + mutation = case.value["mutation"]["id"] + if mutation == "constant-lineage-complete": + target._force_lineage_complete = True + elif mutation == "constant-unknown-coverage": + target._force_unknown = True + elif mutation == "duplicate-preserved-inputs": + target._duplicate_inputs = True + elif mutation == "empty-receipt": + target._empty_receipt = True + elif mutation == "retire-entire-store": + target._retire_all = True + else: + raise ConformanceInputError(f"unknown mutation: {mutation}") + + def observe( + self, case: AdapterCase, importer: object, adapter: TracingAdapter, + context: dict[str, Any], checkpoint: object, receipt: object, + ) -> dict[str, Any]: + target: ReferenceConformanceAdapter = context["adapter"] + blob = "{}" if target._empty_receipt else json.dumps( + receipt.to_dict(), sort_keys=True + ) + triad = dict(receipt.triad) + multiplicity = compare_proposition_multiplicity( + context["before_observations"], self.preserved_observations(target) + ) + preserved = all( + target.recall(term) + for term in ("quiet restaurants", "moderate budget") + ) + unrelated = bool(target.recall("cat")) + forbidden_absent = ( + "is vegetarian" not in blob if case.value["operation"] == "erase" else None + ) + erased_absent = ( + not bool(target.recall("vegetarian")) + if case.value["operation"] == "erase" else None + ) + if multiplicity == "increased": + triad["preserve"] = "fail" + if target._empty_receipt: + aggregate = "failed" + elif any(value != "pass" for value in triad.values()): + aggregate = "failed" + else: + aggregate = receipt.aggregate.value + return { + "checkpoint": next( + row.coverage for row in checkpoint.adapters if row.name == adapter.name + ), + "aggregate": aggregate, + "triad": triad, + "store": { + "multiplicity": multiplicity, + "erased_absent": erased_absent, + "preserved_present": preserved, + "unrelated_present": unrelated, + }, + "receipt": { + "names_store": adapter.name in blob, + "non_trivial": len(blob) > 200, + "forbidden_absent": forbidden_absent, + }, + } + + +def _run_case( + binding: ReferenceConformanceBinding, case: AdapterCase, *, mutate: bool +) -> tuple[dict[str, Any], tuple[str, ...]]: + importer, adapter, context = binding.build(case) + if mutate: + binding.apply_mutation(case, importer, adapter, context) + erratum = binding.erratum(case, context) + adapter.reset_calls() + checkpoint = importer.quarantine(erratum) + receipt = importer.repair_quarantined(erratum, checkpoint) + calls = adapter.calls + observed = binding.observe(case, importer, adapter, context, checkpoint, receipt) + return observed, calls + + +def _anti_vacuity_receipt( + corpus: AdapterCorpus, receipt_validator: Callable[[object], tuple[str, ...]] +) -> str: + mutation = next( + item for item in corpus.validator_controls if item["mutation"] == "empty-receipt" + ) + errors = tuple(receipt_validator({})) + return mutation["required_failure"] if mutation["required_failure"] in errors else "" + + +def _anti_vacuity_feed( + corpus: AdapterCorpus, feed_verifier: Callable[..., list[object]] +) -> str: + from prototype.errata import Erratum, FeedError, Operation, RootRegistry + from prototype.signing import Ed25519Signer + + mutation = next( + item for item in corpus.validator_controls + if item["mutation"] == "no-op-feed-verification" + ) + owner = Ed25519Signer(b"conformance-validator-feed") + event = owner.sign_erratum( + Erratum( + erratum_id="anti-vacuity-gap", + sequence=2, + target_root="fact:diet", + operation=Operation.SUPERSEDE, + valid_from="2026-08-01T00:00:00Z", + replacement="eats meat again", + postconditions={ + "negative": "vegetarian", + "positive": "eats meat again", + "preserve": "quiet restaurants", + }, + ) + ) + try: + feed_verifier([event], owner=owner.public, roots=RootRegistry({"fact:diet"})) + except FeedError as error: + expected = ( + "anti-vacuity-gap: gap at sequence 2, expected 1. " + "A missing erratum may be the one that retired the state this importer " + "is about to serve." + ) + return mutation["required_failure"] if str(error) == expected else "" + except Exception: + return "" + return "" + + +def _semantic_fixture(source_root: Path, case_name: str): + from prototype.semantic import ( + RecordedSemanticVerifier, SemanticObservation, SemanticProbe, VerifierConfig, + ) + + semantic_root = source_root / "spec" / "semantic" + probes_payload = json.loads((semantic_root / "probes.json").read_text())["cases"] + observations_payload = json.loads( + (semantic_root / "observations.json").read_text() + )["cases"] + config = VerifierConfig.from_dict( + json.loads((semantic_root / "verifier-config.json").read_text()) + ) + probes = tuple(SemanticProbe.from_dict(item) for item in probes_payload[case_name]) + observations = tuple( + SemanticObservation.from_dict(item) for item in observations_payload[case_name] + ) + return probes, config, RecordedSemanticVerifier(observations) + + +def _anti_vacuity_semantic( + corpus: AdapterCorpus, source_root: Path, semantic_runner_factory: Callable[[], object] +) -> str: + from prototype.semantic import SemanticCoverage + + mutation = next( + item for item in corpus.validator_controls + if item["mutation"] == "constant-unknown-aggregator" + ) + runner = semantic_runner_factory() + coverages = { + runner.run(*_semantic_fixture(source_root, case_name)).coverage + for case_name in ("verified-correction", "failed-supersession", "unknown-erasure") + } + required = { + SemanticCoverage.VERIFIED, SemanticCoverage.FAILED, SemanticCoverage.UNKNOWN, + } + return mutation["required_failure"] if coverages == required else "" + + +def run_validator_anti_vacuity_controls( + corpus: AdapterCorpus, + source_root: Path, + *, + receipt_validator: Callable[[object], tuple[str, ...]] | None = None, + feed_verifier: Callable[..., list[object]] | None = None, + semantic_runner_factory: Callable[[], object] | None = None, +) -> tuple[ControlResult, ...]: + """Install declared flattering mutations against actual acceptance seams.""" + + from prototype.errata import verify_feed + from prototype.receipts import receipt_acceptance_errors + from prototype.semantic import SemanticProbeRunner + + receipt = receipt_validator or receipt_acceptance_errors + feed = feed_verifier or verify_feed + semantic = semantic_runner_factory or SemanticProbeRunner + failures = ( + _anti_vacuity_receipt(corpus, receipt), + _anti_vacuity_feed(corpus, feed), + _anti_vacuity_semantic(corpus, source_root, semantic), + ) + return tuple( + ControlResult(control["id"], failure == control["required_failure"], failure) + for control, failure in zip(corpus.validator_controls, failures) + ) + + +def _runtime_identity(root: Path) -> tuple[str, str]: + status = _git(root, "status", "--porcelain=v1", "--untracked-files=all") + if status.strip(): + raise ConformanceInputError("runtime source tree is dirty") + commit = _git(root, "rev-parse", "HEAD").decode().strip() + tree = _git(root, "rev-parse", "HEAD^{tree}").decode().strip() + if not re.fullmatch(r"[0-9a-f]{40}", commit) or not re.fullmatch( + r"[0-9a-f]{40}", tree + ): + raise ConformanceInputError("runtime source identity is invalid") + return commit, tree + + +def _binding_source( + binding_factory: Callable[[], ReferenceConformanceBinding], + binding_root: Path | None, +) -> dict[str, str]: + try: + file_path = Path(inspect.getsourcefile(binding_factory) or "").resolve() + root = binding_root.resolve() if binding_root is not None else Path( + _git(file_path.parent, "rev-parse", "--show-toplevel").decode().strip() + ).resolve() + relative = file_path.relative_to(root).as_posix() + payload = file_path.read_bytes() + except (OSError, TypeError, ValueError) as error: + raise ConformanceInputError( + "binding source is not inside a clean Git repository" + ) from error + if relative.startswith(".git/"): + raise ConformanceInputError("binding source is not executable repository source") + tracked = set( + _git(root, "ls-files", "--cached").decode("utf-8").splitlines() + ) + if relative not in tracked: + raise ConformanceInputError("binding source is not tracked by the runtime tree") + commit, tree = _runtime_identity(root) + return { + "commit": commit, + "tree": tree, + "path": relative, + "sha256": hashlib.sha256(payload).hexdigest(), + } + + +@contextmanager +def load_binding_factory( + value: str, + binding_root: Path, +) -> Any: + """Keep an admitted package namespace isolated for the complete run.""" + + try: + module_name, object_name = value.split(":", 1) + except ValueError as error: + raise ConformanceInputError( + "binding must be an importable module:factory" + ) from error + identifier = re.compile(r"^[A-Za-z_]\w*$") + parts = module_name.split(".") + if not parts or not all(identifier.fullmatch(part) for part in parts) or not identifier.fullmatch( + object_name + ): + raise ConformanceInputError("binding must be an importable module:factory") + + root = binding_root.resolve() + module_path = root.joinpath(*parts).with_suffix(".py") + package_path = root.joinpath(*parts, "__init__.py") + candidates = [path for path in (module_path, package_path) if path.is_file()] + if len(candidates) != 1: + raise ConformanceInputError("binding module must resolve to one repository file") + admitted_path = candidates[0].resolve() + relative = admitted_path.relative_to(root).as_posix() + tracked = set(_git(root, "ls-files", "--cached").decode("utf-8").splitlines()) + if relative not in tracked: + raise ConformanceInputError("binding source is not tracked by the runtime tree") + commit, tree = _runtime_identity(root) + admitted_bytes = admitted_path.read_bytes() + source = { + "commit": commit, + "tree": tree, + "path": relative, + "sha256": hashlib.sha256(admitted_bytes).hexdigest(), + } + + executed_sources: dict[str, tuple[str, bytes]] = {} + + class AdmittedLoader(importlib.abc.Loader): + def __init__(self, fullname: str, path: Path, payload: bytes) -> None: + self.fullname = fullname + self.path = path + self.payload = payload + + def create_module(self, spec: object) -> None: + return None + + def exec_module(self, module: object) -> None: + relative_path = self.path.relative_to(root).as_posix() + executed_sources[self.fullname] = (relative_path, self.payload) + code = compile(self.payload, str(self.path), "exec") + exec(code, module.__dict__) + + top_package = parts[0] + + class AdmittedFinder(importlib.abc.MetaPathFinder): + def find_spec( + self, fullname: str, path: object = None, target: object = None + ) -> object: + if fullname != top_package and not fullname.startswith(f"{top_package}."): + return None + name_parts = fullname.split(".") + source_path = root.joinpath(*name_parts).with_suffix(".py") + init_path = root.joinpath(*name_parts, "__init__.py") + matches = [item.resolve() for item in (source_path, init_path) if item.is_file()] + if len(matches) != 1: + raise ImportError(f"binding dependency {fullname!r} is ambiguous or missing") + selected = matches[0] + selected_relative = selected.relative_to(root).as_posix() + if selected_relative not in tracked: + raise ImportError(f"binding dependency {fullname!r} is not tracked") + payload = admitted_bytes if selected == admitted_path else selected.read_bytes() + loader = AdmittedLoader(fullname, selected, payload) + is_package = selected.name == "__init__.py" + return importlib.util.spec_from_file_location( + fullname, + selected, + loader=loader, + submodule_search_locations=[str(selected.parent)] if is_package else None, + ) + + namespace_prefix = f"{top_package}." + saved_modules = { + name: module + for name, module in tuple(sys.modules.items()) + if name == top_package or name.startswith(namespace_prefix) + } + for name in saved_modules: + sys.modules.pop(name, None) + finder = AdmittedFinder() + sys.meta_path.insert(0, finder) + + try: + try: + module = _run_with_timeout(lambda: importlib.import_module(module_name)) + factory = getattr(module, object_name) + except _BindingTimeout as error: + raise ConformanceInputError("binding import timed out") from error + except BaseException as error: + if isinstance(error, KeyboardInterrupt): + raise + raise ConformanceInputError( + f"binding import failed: {type(error).__name__}" + ) from error + if not callable(factory): + raise ConformanceInputError("binding factory must be callable") + try: + loaded_path = Path(inspect.getsourcefile(factory) or "").resolve() + except (OSError, TypeError, ValueError) as error: + raise ConformanceInputError("binding factory has no admitted source") from error + if loaded_path != admitted_path: + raise ConformanceInputError("binding factory source differs from admitted module") + + yield factory, source + + post_commit, post_tree = _runtime_identity(root) + if post_commit != commit or post_tree != tree: + raise ConformanceInputError("binding repository identity changed during execution") + dependency_digest = hashlib.sha256() + for fullname, (dependency_path, payload) in sorted(executed_sources.items()): + dependency_digest.update(fullname.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(dependency_path.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(hashlib.sha256(payload).digest()) + dependency_digest.update(b"\0") + source["dependency_manifest_sha256"] = dependency_digest.hexdigest() + finally: + if finder in sys.meta_path: + sys.meta_path.remove(finder) + for name in tuple(sys.modules): + if name == top_package or name.startswith(namespace_prefix): + sys.modules.pop(name, None) + sys.modules.update(saved_modules) + + +class _BindingTimeout(Exception): + pass + + +def _run_with_timeout(action: Callable[[], Any]) -> Any: + if threading.current_thread() is not threading.main_thread(): + raise ConformanceInputError( + "binding execution requires the main thread for timeout enforcement" + ) + + def expire(signum: int, frame: object) -> None: + raise _BindingTimeout + + prior = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, BINDING_TIMEOUT_SECONDS) + try: + return action() + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, prior) + + +def validate_adapter_conformance( + corpus_path: Path, + source_root: Path, + binding_factory: Callable[[], ReferenceConformanceBinding], + binding_root: Path | None = None, + binding_source: dict[str, str] | None = None, +) -> ConformanceReport: + runtime_commit, runtime_tree = _runtime_identity(source_root) + corpus = load_corpus(corpus_path, source_root, require_canonical_path=True) + source_identity = binding_source or _binding_source(binding_factory, binding_root) + try: + binding = _run_with_timeout(binding_factory) + except _BindingTimeout as error: + raise ConformanceInputError("binding construction timed out") from error + except ConformanceInputError: + raise + except Exception as error: + raise ConformanceInputError( + f"binding construction failed: {type(error).__name__}" + ) from error + try: + binding_name = _run_with_timeout(lambda: binding.name) + except _BindingTimeout as error: + raise ConformanceInputError("binding metadata timed out") from error + except ConformanceInputError: + raise + except Exception as error: + raise ConformanceInputError( + f"binding metadata failed: {type(error).__name__}" + ) from error + if not isinstance(binding_name, str) or not binding_name.strip(): + raise ConformanceInputError("binding metadata name must be non-empty") + results = [] + for case in corpus.cases: + try: + observed, calls = _run_with_timeout( + lambda case=case: _run_case(binding, case, mutate=False) + ) + failures = compare_complete_outcome(case.value["expected"], observed) + missing = tuple(sorted(set(case.value["required_calls"]) - set(calls))) + except _BindingTimeout as error: + raise ConformanceInputError("binding execution timed out") from error + except Exception as error: + raise ConformanceInputError( + f"binding execution failed for {case.case_id}: {type(error).__name__}" + ) from error + mutation_observed = None + mutation_failures: tuple[str, ...] + try: + mutation_observed, _ = _run_with_timeout( + lambda case=case: _run_case(binding, case, mutate=True) + ) + mutation_failures = compare_complete_outcome( + case.value["mutation"]["exact_counter_result"], mutation_observed + ) + except _BindingTimeout as error: + raise ConformanceInputError("binding execution timed out") from error + except Exception as error: + raise ConformanceInputError( + f"binding execution failed for {case.case_id} mutation: " + f"{type(error).__name__}" + ) from error + results.append( + CaseResult( + case_id=case.case_id, + expectation_met=not failures, + failures=failures, + required_calls=tuple(case.value["required_calls"]), + observed_calls=tuple(sorted(set(calls))), + missing_calls=missing, + positive_control_passed=not missing, + mutation_control_passed=not mutation_failures, + mutation_failures=mutation_failures, + observed=observed, + mutation_observed=mutation_observed, + ) + ) + return ConformanceReport( + binding=binding_name, + normative_commit=corpus.normative_target.commit, + normative_surface_digest=corpus.normative_target.surface_digest, + corpus_sha256=corpus.sha256, + runtime_commit=runtime_commit, + runtime_tree=runtime_tree, + binding_source=source_identity, + cases=tuple(results), + validator_controls=run_validator_anti_vacuity_controls(corpus, source_root), + provenance=corpus.provenance, + evidence_boundary=corpus.evidence_boundary, + ) diff --git a/prototype/receipts.py b/prototype/receipts.py index 2f0830d..92644f8 100644 --- a/prototype/receipts.py +++ b/prototype/receipts.py @@ -20,9 +20,18 @@ from typing import Any, Mapping, Sequence from prototype.adapters import Coverage +from prototype.schema import load as load_schema, validate as validate_schema from prototype.signing import VerificationKey +def receipt_acceptance_errors(value: object) -> tuple[str, ...]: + """Return production receipt-acceptance errors, including non-vacuity.""" + + if not isinstance(value, dict) or not value: + return ("receipt is vacuous",) + return tuple(validate_schema(value, load_schema("receipt"))) + + def aggregate_coverage( stores: Mapping[str, Coverage], triad: Mapping[str, str] ) -> Coverage: diff --git a/prototype/workspace.py b/prototype/workspace.py index c247065..9814982 100644 --- a/prototype/workspace.py +++ b/prototype/workspace.py @@ -25,6 +25,10 @@ from prototype.signing import Ed25519Signer, VerificationKey +class ReceiptReadError(ValueError): + """A persisted receipt cannot be parsed as JSON evidence.""" + + class Workspace: def __init__(self, root: Path) -> None: self.root = Path(root) @@ -238,10 +242,16 @@ def _atomic_json(self, path: Path, payload: object) -> None: def all_receipts(self) -> list[tuple[str, dict[str, Any]]]: if not self.receipts_dir.is_dir(): return [] - return [ - (p.name, json.loads(p.read_text(encoding="utf-8"))) - for p in sorted(self.receipts_dir.glob("*.json")) - ] + receipts = [] + for path in sorted(self.receipts_dir.glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ReceiptReadError( + f"{path.name}: receipt JSON is unreadable" + ) from error + receipts.append((path.name, payload)) + return receipts def latest_receipt(self) -> dict[str, Any] | None: receipts = self.all_receipts() diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index 36d4b2e..467d7e9 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -1,8 +1,8 @@ { "schema_version": 1, - "project_version": "0.3.0", + "project_version": "0.4.0", "verdict": "NOT_PROD_READY", - "last_reviewed": "2026-08-12", + "last_reviewed": "2026-08-13", "gates": [ { "id": "G1", @@ -39,6 +39,9 @@ {"kind": "repository", "ref": "spec/semantic/observations.json"}, {"kind": "repository", "ref": "spec/vectors/protocol-manifest.json"}, {"kind": "repository", "ref": "spec/vectors/receipt-binding-mutations.json"}, + {"kind": "repository", "ref": "spec/adapter-conformance.json"}, + {"kind": "repository", "ref": "prototype/conformance.py"}, + {"kind": "repository", "ref": "tests/test_conformance.py"}, {"kind": "repository", "ref": "prototype/README.md"}, {"kind": "repository", "ref": "spec/README.md"}, {"kind": "repository", "ref": "REVIEW_REQUEST.md"}, diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index ce8f4e5..0464da0 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -43,6 +43,7 @@ "tests/test_adapters.py", "tests/test_checkpoints.py", "tests/test_cli.py", + "tests/test_conformance.py", "tests/test_controller.py", "tests/test_ed25519.py", "tests/test_errata_feed.py", @@ -93,8 +94,9 @@ G2_MATRIX_CURRENT_EVIDENCE = ( "Phase 2 implementation includes conflict-disclosed remediation for split-view " "equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract " - "completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, " - "and receipt binding; no qualifying independent review is recorded." + "completeness, plus schemas, semantic probes, adapter-level conformance, validator " + "anti-vacuity controls, key rotation, invalid-target, confidentiality, and receipt binding; " + "no qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( "Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation." @@ -186,6 +188,7 @@ def g2_surface_files(root: Path = ROOT) -> tuple[str, ...]: tuple(sorted((root / "prototype").glob("*.py"))), (root / "prototype" / "README.md",), (root / "spec" / "README.md",), + (root / "spec" / "adapter-conformance.json",), tuple(sorted((root / "spec").glob("*.schema.json"))), tuple(sorted((root / "spec" / "vectors").glob("*.json"))), tuple(sorted((root / "spec" / "semantic").glob("*.json"))), @@ -220,8 +223,20 @@ def g2_surface_digest(root: Path = ROOT) -> str: def g2_surface_digest_at_commit(commit: str, root: Path = ROOT) -> str: if not reviewed_commit_exists(commit, root): raise OSError("reviewed commit is unavailable") + listed = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", commit], + cwd=root, + capture_output=True, + check=False, + text=True, + ) + if listed.returncode != 0: + raise OSError("reviewed commit tree is unavailable") + commit_files = set(listed.stdout.splitlines()) entries = [] for relative in g2_surface_files(root): + if relative == "spec/adapter-conformance.json" and relative not in commit_files: + continue result = subprocess.run( ["git", "show", f"{commit}:{relative}"], cwd=root, capture_output=True, check=False ) diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 9fc95ec..db53930 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -394,6 +394,24 @@ def check_github_actions_runtime(reporter: Reporter) -> None: "Restore immutable Node 24 action pins. " + "; ".join(violations), ) + validate_workflow = ROOT / ".github" / "workflows" / "validate.yml" + validate_text = read_utf8(validate_workflow) if validate_workflow.is_file() else "" + checkout_step = re.search( + rf"(?ms)^\s{{6}}- name: Check out the repository\s*$\n" + rf"^\s{{8}}uses: actions/checkout@{GITHUB_ACTION_PINS['actions/checkout']}[^\n]*$\n" + rf"^\s{{8}}with:\s*$\n" + rf"(?:(?:^\s{{10}}#.*$\n)*)" + rf"^\s{{10}}fetch-depth:\s*0\s*$", + validate_text, + ) + reporter.check( + "GitHub Actions source history", + checkout_step is not None, + "validation checkout fetches immutable Git history required by conformance", + "Set actions/checkout fetch-depth to 0 in validate.yml so historical " + "normative commits are available to fail-closed conformance checks.", + ) + def check_publication_metadata(reporter: Reporter) -> None: readme_path = ROOT / "README.md" @@ -593,23 +611,29 @@ def check_document_version_alignment( cells = [cell.strip() for cell in row.split("|")] if len(cells) == 4 and cells[0] == cells[-1] == "": supported_rows.append((cells[1], cells[2])) - supported_yes = [ - version - for version, status in supported_rows - if status == "Yes" - ] previous_minor = int(minor) - 1 - required_unsupported_rows = { + released_rows = { + (f"{major}.{minor}.x", "Yes"), (f"{major}.{previous_minor}.x and earlier", "No"), ("Unreleased development revisions", "No"), } + prepared_rows = { + (f"{major}.{minor}.x", f"Yes, after `v{repository_version}` is published"), + ( + f"{major}.{previous_minor}.x", + f"Yes, until `v{repository_version}` is published", + ), + (f"{major}.{previous_minor - 1}.x and earlier", "No"), + ("Unreleased development revisions", "No"), + } + supported_set = frozenset(supported_rows) reporter.check( "SECURITY supported version", - supported_yes == [f"{major}.{minor}.x"] - and required_unsupported_rows.issubset(set(supported_rows)), - f"SECURITY.md supports {major}.{minor}.x", - "Keep exactly one Yes row for VERSION major.minor.x and No rows for " - "the prior-version family and unreleased revisions.", + supported_set in {frozenset(released_rows), frozenset(prepared_rows)} + and len(supported_rows) == len(supported_set), + f"SECURITY.md is either release-prepared for or supports {major}.{minor}.x", + "Use exactly the conditional pre-release transition rows or the final " + "released VERSION row plus prior/unreleased No rows.", ) diff --git a/spec/README.md b/spec/README.md index 06cc4ac..83c013a 100644 --- a/spec/README.md +++ b/spec/README.md @@ -14,6 +14,52 @@ than this one can be built and checked against the same contract. | `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | | `vectors/protocol-manifest.json` | Executable key-rotation, concurrency, invalid-target, and confidentiality cases that JSON Schema cannot express. | | `vectors/receipt-binding-mutations.json` | Valid-domain mutations proving every signed receipt field is bound. | +| `adapter-conformance.json` | Five provider-neutral adapter cases and three executable validator anti-vacuity controls. | + +## Adapter conformance + +`adapter-conformance.json` adds the store-facing half of conformance. Every +case binds an immutable normative source and exact quotation, declares the +complete expected checkpoint, aggregate, triad, store, and receipt outcome, +requires calls through the exact adapter instance, and names one flattering +mutation with its exact counter-result. An exception is not evidence that a +mutation was caught. + +Preservation includes bounded proposition multiplicity. Within an inspectable +synthetic conformance scope, an adapter supplies stable provider-local +proposition identities and active assertion counts. Repair must not increase +the count of a preserved proposition unless the erratum requires another +assertion. Text normalization and embedding similarity are not proposition +identity. An adapter unable to expose identity/count for the isolated fixture +reports that observation as `unknown`; it cannot receive a cardinality pass. + +Run: + +```bash +python3 -m prototype.cli adapter-conformance \ + --corpus spec/adapter-conformance.json \ + --source-root . +``` + +For an independently implemented binding in another checkout, add +`--binding module:factory --binding-root /path/to/its/clean/git/checkout`. +Reports bind the LLM Errata runtime commit/tree, exact corpus SHA-256, and the +binding repository commit/tree plus tracked source path and SHA-256 separately. +Binding-package modules are loaded from admitted tracked bytes rather than any +preloaded module cache, and the report binds their dependency-manifest digest. + +Exit `0` means the supplied binding and validator controls passed this +internal corpus. Exit `1` means a behavioral or mutation control failed. Exit +`2` means source, corpus, binding, or execution evidence is invalid or +inconclusive. No result is G2 or G4 evidence by itself. + +Rastislav Drahos/DanceNitra reported the adapter-coverage gap, +duplicate-preservation counterexample, candidate behaviors, and anti-vacuity +attacks in the MIT-licensed artifact at +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. Inspeximus is a +G4 candidate, so this is interested-party input. The LLM Errata corpus and +validator are independently authored; the external runner and fixtures are +not copied or vendored. Receipt conformance evaluates two independent properties. D1 authenticity verifies the importer and every signable byte. D2 coverage truthfulness checks diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json new file mode 100644 index 0000000..aaf3918 --- /dev/null +++ b/spec/adapter-conformance.json @@ -0,0 +1,165 @@ +{ + "schema_version": 1, + "status": "candidate-internal", + "evidence_boundary": "Passing this corpus is internal conformance evidence. It is not G2 or G4 evidence.", + "normative_target": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "surface_digest": "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" + }, + "provenance": { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored." + }, + "cases": [ + { + "id": "undeclared-derivative-must-not-reach-verified", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "IDEA.md", + "quote": "- **No silent completeness:** missing lineage, inaccessible stores, backups, provider logs, model weights, screenshots, and copied prose with severed lineage remain `partial` or `unknown`." + }, + "scenario": "undeclared-derivative", + "required_calls": ["enumerate", "lineage_complete", "quarantine_coverage", "coverage"], + "expected": { + "checkpoint": "unknown", + "aggregate": "unknown", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "constant-lineage-complete", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "complete-lineage-can-reach-verified", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "prototype/receipts.py", + "quote": "Aggregate success requires every required store to be `verified`." + }, + "scenario": "complete-lineage", + "required_calls": ["enumerate", "lineage_complete", "quarantine_coverage", "coverage"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "constant-unknown-coverage", + "exact_counter_result": { + "checkpoint": "unknown", + "aggregate": "unknown", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "repair-preserves-proposition-multiplicity", + "operation": "supersede", + "normative": { + "commit": "333ffcc98c0b8bd5521906357e7454bd61a0d2f0", + "path": "docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md", + "quote": "Within a declared, inspectable adapter scope, repair must not increase the\nactive multiplicity of a preserved proposition unless the erratum explicitly\nrequires an additional assertion." + }, + "scenario": "complete-lineage", + "required_calls": ["repair_inputs", "retire", "rebuild"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "duplicate-preserved-inputs", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "positive": "pass", "preserve": "fail"}, + "store": {"multiplicity": "increased", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "erasure-evidence-is-content-free-and-non-vacuous", + "operation": "erase", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "spec/README.md", + "quote": "Erasure fixtures carry only fixed content-free protocol labels, timestamps,\nverdicts, and digests. They never contain a retired value or raw provider\nresponse." + }, + "scenario": "erasure", + "required_calls": ["retire", "recall", "snapshot", "dispositions"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": true, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": true} + }, + "mutation": { + "id": "empty-receipt", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": true, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": false, "non_trivial": false, "forbidden_absent": true} + } + } + }, + { + "id": "collateral-survives-complete-supersession", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "IDEA.md", + "quote": "Erasure has no positive replacement, but still needs negative and preservation checks. The triad defeats two cheap tricks: adding a new fact while still retrieving the old one, and “fixing” the problem by wiping the whole profile." + }, + "scenario": "complete-lineage", + "required_calls": ["enumerate", "recall", "retire"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "retire-entire-store", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "positive": "fail", "preserve": "fail"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": false, "unrelated_present": false}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + } + ], + "validator_controls": [ + {"id": "empty-receipt-must-fail", "mutation": "empty-receipt", "required_failure": "receipt is vacuous"}, + {"id": "no-op-feed-verification-must-fail", "mutation": "no-op-feed-verification", "required_failure": "accepted feed is incomplete"}, + {"id": "constant-unknown-aggregator-must-fail", "mutation": "constant-unknown-aggregator", "required_failure": "semantic verdict diversity is missing"} + ] +} diff --git a/tests/test_cli.py b/tests/test_cli.py index f777127..87d62e4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,6 +14,9 @@ import unittest from pathlib import Path +from prototype.conformance import ReferenceConformanceBinding +from prototype.receipts import receipt_acceptance_errors + REPO_ROOT = Path(__file__).resolve().parents[1] SEMANTIC_FIXTURES = REPO_ROOT / "spec" / "semantic" @@ -23,6 +26,13 @@ EXIT_INCONCLUSIVE = 2 +class ExplodingConformanceBinding(ReferenceConformanceBinding): + name = "exploding-test-binding" + + def apply_mutation(self, case, importer, adapter, context) -> None: + raise RuntimeError("deliberate mutation failure") + + class CliCase(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory(prefix="errata-cli-") @@ -31,14 +41,19 @@ def setUp(self) -> None: def tearDown(self) -> None: self._tmp.cleanup() - def run_cli(self, *args: str) -> subprocess.CompletedProcess[str]: + def run_cli( + self, *args: str, extra_pythonpath: Path | None = None + ) -> subprocess.CompletedProcess[str]: + pythonpath = str(REPO_ROOT) + if extra_pythonpath is not None: + pythonpath = f"{extra_pythonpath}:{pythonpath}" return subprocess.run( [sys.executable, "-m", "prototype.cli", "--workspace", ".errata", *args], cwd=self.cwd, capture_output=True, text=True, check=False, - env={"PYTHONPATH": str(REPO_ROOT), "PATH": "/usr/bin:/bin"}, + env={"PYTHONPATH": pythonpath, "PATH": "/usr/bin:/bin"}, ) def seed(self) -> None: @@ -64,6 +79,77 @@ def quarantine_and_repair(self) -> subprocess.CompletedProcess[str]: return self.run_cli("repair") +class AdapterConformanceCommand(CliCase): + def run_conformance( + self, *extra: str, extra_pythonpath: Path | None = None + ) -> subprocess.CompletedProcess[str]: + return self.run_cli( + "adapter-conformance", + "--corpus", str(REPO_ROOT / "spec" / "adapter-conformance.json"), + "--source-root", str(REPO_ROOT), + *extra, + extra_pythonpath=extra_pythonpath, + ) + + def test_reference_binding_emits_canonical_passing_report(self) -> None: + result = self.run_conformance() + self.assertEqual(result.returncode, EXIT_OK, result.stdout + result.stderr) + payload = json.loads(result.stdout) + self.assertTrue(payload["passed"]) + self.assertEqual(len(payload["cases"]), 5) + self.assertEqual(len(payload["validator_controls"]), 3) + self.assertRegex(payload["corpus_sha256"], r"^[0-9a-f]{64}$") + self.assertIn("not G2 or G4 evidence", payload["evidence_boundary"]) + self.assertEqual(result.stdout.strip(), json.dumps( + payload, sort_keys=True, separators=(",", ":") + )) + + def test_binding_execution_fault_exits_two(self) -> None: + result = self.run_conformance( + "--binding", "tests.test_cli:ExplodingConformanceBinding" + ) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertEqual(result.stdout, "") + self.assertIn("invalid conformance evidence", result.stderr) + + def test_invalid_source_evidence_exits_two(self) -> None: + result = self.run_cli( + "adapter-conformance", + "--corpus", str(REPO_ROOT / "spec" / "adapter-conformance.json"), + "--source-root", str(self.cwd), + ) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertIn("invalid conformance evidence", result.stderr) + + def test_sourceless_binding_factory_exits_two_without_traceback(self) -> None: + result = self.run_conformance("--binding", "builtins:dict") + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertEqual(result.stdout, "") + self.assertNotIn("Traceback", result.stderr) + + def test_crashing_binding_import_exits_two_without_traceback(self) -> None: + binding_root = self.cwd / "binding" + binding_root.mkdir() + (binding_root / "broken.py").write_text( + 'raise RuntimeError("top-level failure")\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "CLI Tests"), + ("git", "add", "broken.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=binding_root, check=True) + result = self.run_conformance( + "--binding", "broken:factory", "--binding-root", str(binding_root), + extra_pythonpath=binding_root, + ) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertIn("binding import failed: RuntimeError", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + class WorkspaceLifecycle(CliCase): def test_commands_refuse_before_init(self) -> None: result = self.run_cli("repair") @@ -182,12 +268,33 @@ def test_plan_changes_nothing(self) -> None: class ReceiptsAreVerifiable(CliCase): + def test_empty_receipt_is_rejected_by_shared_acceptance_seam(self) -> None: + self.assertIn("receipt is vacuous", receipt_acceptance_errors({})) + def test_a_genuine_receipt_verifies(self) -> None: self.seed() self.publish_supersession() self.quarantine_and_repair() self.assertEqual(self.run_cli("verify").returncode, EXIT_OK) + def test_empty_receipt_is_refused_without_traceback(self) -> None: + self.seed() + path = self.cwd / ".errata" / "receipts" / "empty.json" + path.write_text("{}", encoding="utf-8") + result = self.run_cli("verify") + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertIn("receipt is vacuous", result.stdout) + self.assertNotIn("Traceback", result.stderr) + + def test_malformed_receipt_json_is_refused_without_traceback(self) -> None: + self.seed() + path = self.cwd / ".errata" / "receipts" / "malformed.json" + path.write_text("{not json", encoding="utf-8") + result = self.run_cli("verify") + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertIn("receipt JSON is unreadable", result.stdout) + self.assertNotIn("Traceback", result.stderr) + def test_a_tampered_aggregate_is_caught(self) -> None: self.seed() self.publish_supersession() diff --git a/tests/test_conformance.py b/tests/test_conformance.py new file mode 100644 index 0000000..704644a --- /dev/null +++ b/tests/test_conformance.py @@ -0,0 +1,674 @@ +"""Adapter-conformance corpus and validator controls. + +These tests use public validator seams. The external candidate fixture is not +imported or copied: each expectation is derived from the LLM Errata contract. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import time +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +from prototype.conformance import ( + ConformanceInputError, + PropositionObservation, + ReferenceConformanceAdapter, + ReferenceConformanceBinding, + TracingAdapter, + compare_complete_outcome, + compare_proposition_multiplicity, + load_corpus, + load_binding_factory, + run_validator_anti_vacuity_controls, + validate_adapter_conformance, +) + + +ROOT = Path(__file__).resolve().parents[1] +CORPUS = ROOT / "spec" / "adapter-conformance.json" + + +class CorpusValidation(unittest.TestCase): + def changed_corpus(self, change) -> Path: + payload = json.loads(CORPUS.read_text(encoding="utf-8")) + change(payload) + directory = tempfile.TemporaryDirectory(prefix="errata-corpus-") + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "corpus.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def test_checked_in_corpus_binds_immutable_normative_sources(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + self.assertEqual(corpus.schema_version, 1) + self.assertEqual( + corpus.normative_target.commit, + "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + ) + self.assertEqual(len(corpus.cases), 5) + self.assertEqual(len(corpus.validator_controls), 3) + + def test_wrong_surface_digest_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["normative_target"].__setitem__("surface_digest", "0" * 64) + ) + with self.assertRaisesRegex(ConformanceInputError, "surface digest"): + load_corpus(path, ROOT) + + def test_noncanonical_corpus_path_is_refused_for_execution(self) -> None: + path = self.changed_corpus(lambda value: None) + with self.assertRaisesRegex(ConformanceInputError, "canonical checked-in"): + load_corpus(path, ROOT, require_canonical_path=True) + + def test_normative_source_must_be_reachable_from_runtime_history(self) -> None: + path = self.changed_corpus( + lambda value: value["cases"][0]["normative"].__setitem__( + "commit", "f" * 40 + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "reachable"): + load_corpus(path, ROOT) + + def test_new_current_surface_files_do_not_change_historical_manifest(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + self.assertEqual( + corpus.normative_target.surface_digest, + "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12", + ) + + def test_quotation_drift_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["cases"][0]["normative"].__setitem__( + "quote", "Missing lineage is always verified." + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "quotation"): + load_corpus(path, ROOT) + + def test_partial_expected_outcome_is_refused(self) -> None: + def remove_aggregate(value) -> None: + del value["cases"][0]["expected"]["aggregate"] + + with self.assertRaisesRegex(ConformanceInputError, "expected outcome"): + load_corpus(self.changed_corpus(remove_aggregate), ROOT) + + def test_empty_provenance_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["provenance"].__setitem__("source_url", "") + ) + with self.assertRaisesRegex(ConformanceInputError, "provenance"): + load_corpus(path, ROOT) + + def test_every_provenance_field_is_bound_to_the_accepted_contribution(self) -> None: + expected = { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored.", + } + self.assertEqual(load_corpus(CORPUS, ROOT).provenance, expected) + for field in expected: + with self.subTest(field=field): + path = self.changed_corpus( + lambda value, field=field: value["provenance"].__setitem__( + field, "plausible but wrong" + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "provenance"): + load_corpus(path, ROOT) + + +class TargetTracing(unittest.TestCase): + def test_only_calls_through_wrapped_target_are_recorded(self) -> None: + class Adapter: + name = "target" + + def coverage(self, root: str) -> str: + return root + + adapter = Adapter() + traced = TracingAdapter(adapter) + + def coverage(root: str) -> str: + return root + + coverage("unrelated") + self.assertEqual(traced.calls, ()) + self.assertEqual(traced.coverage("root"), "root") + self.assertEqual(traced.calls, ("coverage",)) + self.assertIs(traced.target, adapter) + + def test_attribute_reads_do_not_count_as_method_calls(self) -> None: + class Adapter: + name = "target" + + traced = TracingAdapter(Adapter()) + self.assertEqual(traced.name, "target") + self.assertEqual(traced.calls, ()) + + def test_trace_window_can_exclude_fixture_and_observation_calls(self) -> None: + class Adapter: + def coverage(self, root: str) -> str: + return root + + traced = TracingAdapter(Adapter()) + traced.coverage("fixture") + traced.reset_calls() + traced.coverage("controller") + calls = traced.calls + traced.coverage("observation") + self.assertEqual(calls, ("coverage",)) + self.assertEqual(traced.calls, ("coverage", "coverage")) + + +class CompleteComparison(unittest.TestCase): + def outcome(self) -> dict[str, object]: + return { + "checkpoint": "verified", + "aggregate": "verified", + "triad": { + "negative": "pass", + "positive": "pass", + "preserve": "pass", + }, + "store": { + "multiplicity": "known", + "erased_absent": None, + "preserved_present": True, + "unrelated_present": True, + }, + "receipt": { + "names_store": True, + "non_trivial": True, + "forbidden_absent": None, + }, + } + + def test_exact_outcome_passes(self) -> None: + value = self.outcome() + self.assertEqual(compare_complete_outcome(value, value), ()) + + def test_extra_aggregate_and_triad_failures_are_rejected(self) -> None: + expected = self.outcome() + observed = self.outcome() + observed["aggregate"] = "failed" + observed["triad"] = { + "negative": "pass", + "positive": "fail", + "preserve": "pass", + } + failures = compare_complete_outcome(expected, observed) + self.assertIn("aggregate: expected 'verified', got 'failed'", failures) + self.assertIn("triad.positive: expected 'pass', got 'fail'", failures) + + def test_missing_or_extra_fields_are_rejected(self) -> None: + expected = self.outcome() + observed = self.outcome() + del observed["receipt"]["names_store"] + observed["store"]["unexpected"] = True + failures = compare_complete_outcome(expected, observed) + self.assertIn("receipt.names_store: missing", failures) + self.assertIn("store.unexpected: unexpected", failures) + + def test_json_scalar_types_must_match_exactly(self) -> None: + for wrong in (1, 1.0): + with self.subTest(wrong=wrong): + expected = self.outcome() + observed = self.outcome() + observed["receipt"]["names_store"] = wrong + self.assertIn( + f"receipt.names_store: expected True, got {wrong!r}", + compare_complete_outcome(expected, observed), + ) + + +class PropositionMultiplicity(unittest.TestCase): + def test_reference_observations_use_stable_provider_local_ids_and_counts(self) -> None: + observations = ReferenceConformanceAdapter().proposition_observations() + self.assertEqual( + observations, + ( + PropositionObservation("fixture:budget", "budget", 2), + PropositionObservation("fixture:diet", "diet", 2), + PropositionObservation("fixture:pet", "pet", 1), + PropositionObservation("fixture:quiet", "quiet", 2), + ), + ) + + def test_text_equivalence_or_substrings_cannot_create_proposition_identity(self) -> None: + adapter = ReferenceConformanceAdapter() + before = adapter.proposition_observations() + adapter._active["text-only-alias"] = ( + "prefix prefers quiet restaurants suffix; moderate budget-ish" + ) + after = adapter.proposition_observations() + self.assertEqual(compare_proposition_multiplicity(before, after), "known") + + def test_unavailable_identity_or_count_is_unknown(self) -> None: + before = (PropositionObservation("fixture:quiet", "quiet", 2),) + self.assertEqual(compare_proposition_multiplicity(before, None), "unknown") + + def test_exact_identity_count_increase_is_detected(self) -> None: + before = (PropositionObservation("fixture:quiet", "quiet", 2),) + after = (PropositionObservation("fixture:quiet", "quiet", 3),) + self.assertEqual(compare_proposition_multiplicity(before, after), "increased") + + +class AdapterCases(unittest.TestCase): + def test_reference_binding_passes_five_cases_and_exact_mutations(self) -> None: + report = validate_adapter_conformance( + CORPUS, ROOT, ReferenceConformanceBinding + ) + self.assertTrue(report.passed, report.canonical_json()) + self.assertEqual(len(report.cases), 5) + for result in report.cases: + self.assertTrue(result.expectation_met, result.failures) + self.assertTrue(result.positive_control_passed, result.missing_calls) + self.assertTrue(result.mutation_control_passed, result.mutation_failures) + + def test_binding_execution_fault_is_invalid_evidence(self) -> None: + class ExplodingBinding(ReferenceConformanceBinding): + name = "exploding" + + def apply_mutation(self, case, importer, adapter, context) -> None: + raise RuntimeError("mutation setup broke") + + with self.assertRaisesRegex(ConformanceInputError, "binding execution"): + validate_adapter_conformance(CORPUS, ROOT, ExplodingBinding) + + def test_binding_execution_has_a_hard_timeout(self) -> None: + class SlowBinding(ReferenceConformanceBinding): + def build(self, case): + time.sleep(0.1) + return super().build(case) + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + validate_adapter_conformance(CORPUS, ROOT, SlowBinding) + + def test_binding_constructor_has_a_hard_timeout(self) -> None: + class SlowConstructorBinding(ReferenceConformanceBinding): + def __init__(self): + time.sleep(0.1) + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + validate_adapter_conformance(CORPUS, ROOT, SlowConstructorBinding) + + def test_binding_metadata_has_a_hard_timeout(self) -> None: + class SlowMetadataBinding(ReferenceConformanceBinding): + @property + def name(self): + time.sleep(0.1) + return "slow-metadata" + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex( + ConformanceInputError, "binding metadata timed out" + ): + validate_adapter_conformance(CORPUS, ROOT, SlowMetadataBinding) + + +class RuntimeSourceIdentity(unittest.TestCase): + def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None: + report = validate_adapter_conformance(CORPUS, ROOT, ReferenceConformanceBinding) + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, + capture_output=True, text=True, + ).stdout.strip() + tree = subprocess.run( + ["git", "rev-parse", "HEAD^{tree}"], cwd=ROOT, check=True, + capture_output=True, text=True, + ).stdout.strip() + self.assertEqual(report.runtime_commit, commit) + self.assertEqual(report.runtime_tree, tree) + self.assertEqual(report.binding_source["path"], "prototype/conformance.py") + self.assertRegex(report.binding_source["sha256"], r"^[0-9a-f]{64}$") + self.assertEqual(report.binding_source["commit"], commit) + self.assertEqual(report.binding_source["tree"], tree) + self.assertRegex(report.corpus_sha256, r"^[0-9a-f]{64}$") + + def test_external_binding_can_be_bound_to_its_own_clean_repository(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-") as directory: + binding_root = Path(directory) + source = binding_root / "binding.py" + source.write_text("def factory():\n return None\n", encoding="utf-8") + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "binding.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=binding_root, check=True) + namespace: dict[str, object] = {} + exec(compile(source.read_text(), str(source), "exec"), namespace) + factory = namespace["factory"] + factory.__module__ = "binding" + with patch("prototype.conformance.inspect.getsourcefile", return_value=str(source)): + from prototype.conformance import _binding_source + + identity = _binding_source(factory, binding_root) + self.assertEqual(identity["path"], "binding.py") + self.assertRegex(identity["commit"], r"^[0-9a-f]{40}$") + + def test_sourceless_binding_factory_is_invalid_evidence(self) -> None: + from prototype.conformance import _binding_source + + with self.assertRaisesRegex(ConformanceInputError, "clean Git repository"): + _binding_source(dict, None) + + def test_binding_import_runtime_error_is_invalid_evidence(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + (root / "broken.py").write_text( + 'raise RuntimeError("top-level failure")\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "broken.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + with patch("sys.path", [str(root), *sys.path]): + with self.assertRaisesRegex(ConformanceInputError, "RuntimeError"): + with load_binding_factory("broken:factory", root): + pass + + def test_binding_import_has_a_hard_timeout(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + (root / "slow.py").write_text( + "import time\ntime.sleep(0.1)\ndef factory():\n return None\n", + encoding="utf-8", + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "slow.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + with patch("sys.path", [str(root), *sys.path]), patch( + "prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01 + ): + with self.assertRaisesRegex(ConformanceInputError, "import timed out"): + with load_binding_factory("slow:factory", root): + pass + + def test_binding_executes_the_admitted_bytes_not_a_reopened_path(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + source = root / "stable.py" + source.write_text( + 'def factory():\n return "ADMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "stable.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + original_read = Path.read_bytes + + def substitute_after_read(path: Path) -> bytes: + payload = original_read(path) + if path.resolve() == source.resolve(): + source.write_text( + 'def factory():\n return "SUBSTITUTED"\n', encoding="utf-8" + ) + return payload + + with patch("pathlib.Path.read_bytes", substitute_after_read): + with self.assertRaisesRegex(ConformanceInputError, "dirty"): + with load_binding_factory("stable:factory", root) as (factory, _): + self.assertEqual(factory(), "ADMITTED") + + def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text( + "from .helper import VALUE\n" + "def factory():\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached = types.ModuleType("bindingpkg.helper") + cached.VALUE = "CACHED-UNBOUND" + prior = sys.modules.get("bindingpkg.helper") + sys.modules["bindingpkg.helper"] = cached + try: + with load_binding_factory("bindingpkg:factory", root) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertRegex( + identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" + ) + finally: + if prior is None: + sys.modules.pop("bindingpkg.helper", None) + else: + sys.modules["bindingpkg.helper"] = prior + + def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "subbinding.py").write_text( + "from .helper import VALUE\n" + "def factory():\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached_parent = types.ModuleType("bindingpkg") + cached_parent.__path__ = [str(package)] + cached_helper = types.ModuleType("bindingpkg.helper") + cached_helper.VALUE = "CACHED-UNBOUND" + prior_parent = sys.modules.get("bindingpkg") + prior_helper = sys.modules.get("bindingpkg.helper") + sys.modules["bindingpkg"] = cached_parent + sys.modules["bindingpkg.helper"] = cached_helper + try: + with load_binding_factory( + "bindingpkg.subbinding:factory", root + ) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertIsNot(sys.modules["bindingpkg"], cached_parent) + self.assertIsNot(sys.modules["bindingpkg.helper"], cached_helper) + self.assertIs(sys.modules["bindingpkg"], cached_parent) + self.assertIs(sys.modules["bindingpkg.helper"], cached_helper) + finally: + if prior_parent is None: + sys.modules.pop("bindingpkg", None) + else: + sys.modules["bindingpkg"] = prior_parent + if prior_helper is None: + sys.modules.pop("bindingpkg.helper", None) + else: + sys.modules["bindingpkg.helper"] = prior_helper + self.assertRegex( + identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" + ) + + def test_lazy_dependency_import_remains_isolated_for_factory_lifetime(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "subbinding.py").write_text( + "def factory():\n" + " from .helper import VALUE\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached_parent = types.ModuleType("bindingpkg") + cached_parent.__path__ = [str(package)] + cached_helper = types.ModuleType("bindingpkg.helper") + cached_helper.VALUE = "CACHED-UNBOUND" + saved = {name: sys.modules.get(name) for name in ("bindingpkg", "bindingpkg.helper")} + sys.modules["bindingpkg"] = cached_parent + sys.modules["bindingpkg.helper"] = cached_helper + try: + with load_binding_factory( + "bindingpkg.subbinding:factory", root + ) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertRegex(identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$") + finally: + for name, module in saved.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: + source_root = Path(directory) + subprocess.run(["git", "init", "-q"], cwd=source_root, check=True) + marker = source_root / "tracked.txt" + marker.write_text("clean\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=source_root, check=True) + subprocess.run( + [ + "git", "-c", "user.name=LLM Errata Tests", + "-c", "user.email=tests@example.invalid", + "commit", "-q", "-m", "fixture", + ], + cwd=source_root, + check=True, + ) + marker.write_text("dirty\n", encoding="utf-8") + with self.assertRaisesRegex(ConformanceInputError, "dirty"): + validate_adapter_conformance( + CORPUS, source_root, ReferenceConformanceBinding + ) + + def test_git_timeout_is_invalid_source_evidence(self) -> None: + with patch( + "prototype.conformance.subprocess.run", + side_effect=subprocess.TimeoutExpired(["git"], 5), + ): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + load_corpus(CORPUS, ROOT) + + +class AntiVacuity(unittest.TestCase): + def test_all_declared_validator_attacks_are_rejected(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls(corpus, ROOT) + self.assertEqual( + [item.control_id for item in controls], + [ + "empty-receipt-must-fail", + "no-op-feed-verification-must-fail", + "constant-unknown-aggregator-must-fail", + ], + ) + self.assertTrue(all(item.passed for item in controls), controls) + self.assertEqual( + [item.observed_failure for item in controls], + [ + "receipt is vacuous", + "accepted feed is incomplete", + "semantic verdict diversity is missing", + ], + ) + + def test_permissive_receipt_validator_makes_empty_receipt_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, receipt_validator=lambda value: (), + ) + self.assertFalse(controls[0].passed) + + def test_permissive_production_receipt_schema_makes_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + with patch("prototype.receipts.receipt_acceptance_errors", return_value=()): + controls = run_validator_anti_vacuity_controls(corpus, ROOT) + self.assertFalse(controls[0].passed) + + def test_no_op_feed_verifier_makes_acceptance_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, feed_verifier=lambda errata, **kwargs: list(errata), + ) + self.assertFalse(controls[1].passed) + + def test_unrelated_gap_exception_cannot_satisfy_feed_control(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + + def crash(*args, **kwargs): + raise RuntimeError("unrelated gap in logging") + + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, feed_verifier=crash, + ) + self.assertFalse(controls[1].passed) + + def test_constant_unknown_semantic_runner_makes_diversity_control_fail(self) -> None: + from prototype.semantic import SemanticCoverage + + class ConstantUnknownRunner: + def run(self, probes, config, verifier): + return type("Report", (), {"coverage": SemanticCoverage.UNKNOWN})() + + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, semantic_runner_factory=ConstantUnknownRunner, + ) + self.assertFalse(controls[2].passed) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 011021e..20177e5 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -11,6 +11,7 @@ from pathlib import Path from scripts.check_readiness import ( + G2_MATRIX_CURRENT_EVIDENCE, G6_SCOPE, g2_surface_digest, g2_surface_digest_at_commit, @@ -36,6 +37,32 @@ SCRIPT = "check_readiness.py" +class Release040ReadinessBoundary(unittest.TestCase): + def test_release_updates_version_without_upgrading_external_gates(self) -> None: + root = Path(__file__).resolve().parents[1] + payload = json.loads( + (root / "readiness" / "production-readiness.json").read_text() + ) + self.assertEqual(payload["project_version"], "0.4.0") + self.assertEqual(payload["verdict"], "NOT_PROD_READY") + self.assertEqual( + {gate["id"]: gate["status"] for gate in payload["gates"]}, + { + "G1": "PASS", + "G2": "BLOCKED", + "G3": "BLOCKED", + "G4": "BLOCKED", + "G5": "BLOCKED", + "G6": "BLOCKED", + }, + ) + g2 = next(gate for gate in payload["gates"] if gate["id"] == "G2") + refs = {item.get("ref") for item in g2["evidence"]} + self.assertIn("prototype/conformance.py", refs) + self.assertIn("spec/adapter-conformance.json", refs) + self.assertIn("tests/test_conformance.py", refs) + + class ReadinessCheckerPasses(unittest.TestCase): def test_cryptography_qualification_foregrounds_refusal_evidence(self) -> None: qualification = ( @@ -62,6 +89,7 @@ def test_g2_surface_includes_checkpoint_contract_and_tests(self) -> None: files = set(g2_surface_files()) self.assertIn("prototype/checkpoints.py", files) self.assertIn("tests/test_checkpoints.py", files) + self.assertIn("spec/adapter-conformance.json", files) def test_g6_complete_measured_report_is_commit_and_deployment_bound(self) -> None: with repo_copy() as source, tempfile.TemporaryDirectory() as temp: @@ -277,12 +305,13 @@ def _complete_review(root: Path, commit: str) -> dict[str, object]: def test_consistently_formatted_canonical_matrix_cells_are_accepted(self) -> None: def mutate(root): + version = (root / "VERSION").read_text(encoding="utf-8").strip() path = root / "PRODUCTION_READINESS.md" lines = path.read_text(encoding="utf-8").splitlines() formatted = [] for line in lines: if line.startswith("| Version |"): - formatted.append("| **Version** | `**0.3.0**` |") + formatted.append(f"| **Version** | `**{version}**` |") elif line.startswith("| Verdict |"): formatted.append("| `Verdict` | __**NOT_PROD_READY**__ |") elif line.startswith("| G"): @@ -364,9 +393,10 @@ def test_project_version_drift_is_rejected(self) -> None: def test_matrix_project_version_contradiction_is_rejected(self) -> None: def mutate(root): + version = (root / "VERSION").read_text(encoding="utf-8").strip() rewrite( root / "PRODUCTION_READINESS.md", - "| Version | 0.3.0 |", + f"| Version | {version} |", "| Version | 9.9.9 |", ) @@ -435,7 +465,7 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded.", + G2_MATRIX_CURRENT_EVIDENCE, "local tests prove readiness.", ) @@ -501,7 +531,8 @@ def mutate(root): def test_missing_matrix_project_version_is_rejected(self) -> None: def mutate(root): - rewrite(root / "PRODUCTION_READINESS.md", "| Version | 0.3.0 |\n", "") + version = (root / "VERSION").read_text(encoding="utf-8").strip() + rewrite(root / "PRODUCTION_READINESS.md", f"| Version | {version} |\n", "") result = check_after(SCRIPT, mutate) self.assert_rejected_without_traceback(result, "matrix project version") diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 8941f69..73376ab 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -17,6 +17,27 @@ SCRIPT = "validate_repo.py" +class Release040Metadata(unittest.TestCase): + def test_version_citation_maturity_security_and_changelog_align(self) -> None: + root = Path(__file__).resolve().parents[1] + self.assertEqual((root / "VERSION").read_text().strip(), "0.4.0") + citation = (root / "CITATION.cff").read_text(encoding="utf-8") + self.assertIn("version: 0.4.0", citation) + self.assertIn("date-released: 2026-08-13", citation) + self.assertIn("Version 0.4.0", (root / "README.md").read_text()) + self.assertIn( + "| 0.4.x | Yes, after `v0.4.0` is published |", + (root / "SECURITY.md").read_text(), + ) + changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") + self.assertIn("## [0.4.0]", changelog) + self.assertIn("Rastislav Drahos", changelog) + self.assertIn("2ba1e299b3483b9038d03387345702427608b90b", changelog) + security = (root / "SECURITY.md").read_text(encoding="utf-8") + self.assertIn("beginning with the immutable `v0.4.0` release", security) + self.assertIn("until `v0.4.0` is published", security) + + class ValidatorPasses(unittest.TestCase): def test_unmodified_repository_passes(self) -> None: with repo_copy() as root: @@ -364,7 +385,8 @@ def test_security_supported_version_drift_is_rejected(self) -> None: def mutate(root: Path) -> None: current = (root / "VERSION").read_text(encoding="utf-8").strip() major, minor, _ = current.split(".") - rewrite(root / "SECURITY.md", f"| {major}.{minor}.x | Yes |", "| 0.0.x | Yes |") + expected = f"| {major}.{minor}.x | Yes, after `v{current}` is published |" + rewrite(root / "SECURITY.md", expected, "| 0.0.x | Yes |") result = check_after(SCRIPT, mutate) self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) @@ -374,7 +396,7 @@ def test_extra_security_supported_version_is_rejected(self) -> None: def mutate(root: Path) -> None: current = (root / "VERSION").read_text(encoding="utf-8").strip() major, minor, _ = current.split(".") - supported = f"| {major}.{minor}.x | Yes |" + supported = f"| {major}.{minor}.x | Yes, after `v{current}` is published |" rewrite(root / "SECURITY.md", supported, supported + "\n| 0.2.x | Yes |") result = check_after(SCRIPT, mutate) @@ -383,10 +405,13 @@ def mutate(root: Path) -> None: def test_earlier_security_versions_cannot_be_supported(self) -> None: def mutate(root: Path) -> None: + current = (root / "VERSION").read_text(encoding="utf-8").strip() + major, minor, _ = current.split(".") + row = f"| {major}.{int(minor) - 2}.x and earlier |" rewrite( root / "SECURITY.md", - "| 0.2.x and earlier | No |", - "| 0.2.x and earlier | Yes |", + f"{row} No |", + f"{row} Yes |", ) result = check_after(SCRIPT, mutate) @@ -454,6 +479,12 @@ def test_workflows_pin_node24_action_releases(self) -> None: self.assertIn("actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", workflows) self.assertIn("actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", workflows) + def test_validate_workflow_fetches_immutable_history_for_conformance(self) -> None: + workflow = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "validate.yml" + ).read_text(encoding="utf-8") + self.assertIn("fetch-depth: 0", workflow) + def test_validator_rejects_deprecated_action_major(self) -> None: def mutate(root: Path) -> None: path = root / ".github" / "workflows" / "links.yml" @@ -471,6 +502,26 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("GitHub Actions Node 24 pins", result.stdout) + def test_validator_rejects_shallow_validate_checkout(self) -> None: + def mutate(root: Path) -> None: + rewrite(root / ".github" / "workflows" / "validate.yml", "fetch-depth: 0", "fetch-depth: 1") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("GitHub Actions source history", result.stdout) + + def test_history_guard_ignores_comment_camouflage(self) -> None: + def mutate(root: Path) -> None: + path = root / ".github" / "workflows" / "validate.yml" + text = path.read_text(encoding="utf-8") + text = text.replace("fetch-depth: 0", "fetch-depth: 1", 1) + text += "\n# legacy requirement text: fetch-depth: 0\n" + path.write_text(text, encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("GitHub Actions source history", result.stdout) + if __name__ == "__main__": unittest.main()