From 481d329e1763b658e6cd355bea3e5956c0fa1df3 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 22:48:43 +0200 Subject: [PATCH 01/27] docs: design adapter conformance hardening --- ...-conformance-validator-hardening-design.md | 232 ++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md diff --git a/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md new file mode 100644 index 0000000..726b46a --- /dev/null +++ b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md @@ -0,0 +1,232 @@ +# Adapter Conformance and Validator Hardening Design + +**Date:** 2026-08-13 +**Owner:** Thomas Willner +**Status:** Approved for implementation +**Target release:** 0.4.0 + +## Purpose + +LLM Errata currently exercises its wire contract, controller, semantic +aggregator, and receipt binding more deeply than it exercises independently +authored store adapters. Rastislav Drahos reported that an external runner +could reproduce 28 of 28 published cases while only one case reached any store +adapter. He then published candidate adapter cases and validator anti-vacuity +requirements at immutable commit +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. + +This change adopts the verified behavioral findings without copying or +vendoring the contributed runner. It adds an independently authored, +provider-neutral adapter-conformance corpus and a fail-closed validator whose +own controls must demonstrably fail against flattering implementations. + +Release 0.4.0 remains an experimental conformance proposal and tested +reference implementation. It does not change the project verdict from +`NOT_PROD_READY`, and it does not satisfy G2 or G4. + +## Ownership, licence, and provenance + +- LLM Errata, its repository, specification, project name, and files authored + in this repository remain owned and governed by Thomas Willner. +- Repository publication and versioned releases do not transfer copyright. +- Rastislav Drahos and DanceNitra retain copyright in their candidate fixture. + Its MIT licence permits reuse but does not assign ownership. +- This repository will not copy or vendor that runner or its fixture files. + Implementation is independently authored from the accepted behavioral + requirements and the LLM Errata normative contract. +- Documentation must credit Rastislav Drahos and DanceNitra for the reported + adapter-coverage gap, duplicate-preservation counterexample, candidate cases, + and anti-vacuity findings. It must identify the immutable source commit, + MIT licence, interested-party conflict, and disclosed Claude Opus 5 + co-authorship. +- Existing LLM Errata licence terms remain unchanged. Any product or + commercial implementation remains subject to repository terms, including + accessible credit to LLM Errata and Thomas Willner and any required written + grant. Technical adoption, contributor credit, or release publication does + not imply endorsement, partnership, certification, or commercial permission. + +## Normative preservation rule + +Preservation is strengthened from boolean recallability to bounded +multiplicity preservation. + +Within a declared, inspectable adapter scope, repair must not increase the +active multiplicity of a preserved proposition unless the erratum explicitly +requires an additional assertion. A proposition is identified by the +adapter-provided stable proposition identity used by its conformance +observation interface, not by lossy text normalization or embedding +similarity. + +The conformance observation for each active proposition contains: + +- a stable, provider-local proposition identifier; +- a content-free or synthetic fixture label suitable for comparison; and +- an active assertion count. + +Adapters unable to expose proposition identity or active multiplicity cannot +pass the cardinality case. They report `unknown` for that observation. This +rule is scoped evidence: it does not claim semantic uniqueness outside the +declared adapter surface. + +## Corpus format + +`spec/adapter-conformance.json` is a versioned, provider-neutral corpus. Each +case contains: + +- stable case ID and operation; +- synthetic initial state; +- complete expected checkpoint, aggregate, triad, receipt, and store + observations; +- required adapter method calls; +- a pinned normative source path and exact quotation; +- one named mutation and the complete result that mutation must produce; and +- provenance and evidence-boundary fields. + +Partial expected outcomes are forbidden. A case fails when any extra triad +failure, worse aggregate result, missing store observation, unexpected +exception, or unspecified result appears. + +Initial behavioral cases cover: + +1. undeclared derivatives cannot become `verified`; +2. complete lineage can reach `verified`; +3. a rebuild cannot increase preserved-proposition multiplicity; +4. erasure evidence is content-free and non-vacuous; and +5. collateral survives supersession while the complete repair still passes. + +## Validator architecture + +`prototype/conformance.py` owns parsing, source binding, execution, and +result comparison. It exposes one public entry point: + +`validate_adapter_conformance(corpus_path, source_root, binding_factory) -> ConformanceReport`. + +The validator is standard-library only. + +### Source identity + +The corpus binds: + +- exact LLM Errata Git commit; +- canonical conformance-surface digest; +- fixture schema version; and +- every normative source path and exact quotation. + +Validation refuses a dirty or different source tree, a commit mismatch, a +digest mismatch, a missing source, or quotation drift. Source mismatch is an +invalid run, never a case failure or pass. + +### Target-instance tracing + +Required calls are observed through a proxy around the exact adapter instance +given to the controller. The proxy records protocol member access and calls. +Global frame names, unrelated modules, helper functions, and another object +with the same method name cannot satisfy a positive control. + +### Mutation controls + +Every case runs twice: + +1. honest binding; +2. the declared flattering mutation installed through the public adapter or + validator seam. + +The mutation must complete and produce the exact declared counter-result. +Any unexpected exception, missing result, different failure, or unchanged pass +fails the control. Exceptions count only when the corpus explicitly declares +that exact exception type and message as the required outcome. + +### Validator anti-vacuity controls + +Three executable meta-tests attack the validator: + +- an empty receipt cannot satisfy confidentiality; +- no-op feed verification cannot satisfy accept-side feed cases; and +- a constant-`unknown` aggregator cannot satisfy the semantic corpus. + +These are validator requirements, not adapter scores. A release cannot claim +validator readiness unless all three mutations are rejected for their +specified semantic reason. + +## Reference binding + +The LLM Errata reference binding uses only public adapter/controller +interfaces plus a conformance observation seam. Production adapters are not +required to expose raw private store contents. A binding may expose only +synthetic fixture identities and counts for an isolated conformance run. + +The corpus and validator must run against the repository reference adapter. +Passing the reference binding demonstrates harness behavior, not independent +interoperability. + +## CLI and evidence + +The existing CLI gains an `adapter-conformance` command accepting: + +- corpus path; +- source root; and +- an importable binding factory. + +Output is one canonical JSON report with source identity, per-case results, +positive controls, mutation controls, anti-vacuity results, provenance, and an +explicit statement that the report is not G2 or G4 evidence. + +Exit codes: + +- `0`: corpus valid and every case/control passes; +- `1`: one or more behavioral or mutation controls fail; +- `2`: source, corpus, binding, or execution evidence is invalid or + inconclusive. + +## Release integration + +Version 0.4.0 includes the current `agent/g2-publication` surface plus this +validator hardening. The release: + +- updates `VERSION`, `CITATION.cff`, README maturity text, supported + security version, readiness ledger/matrix, and changelog; +- records contributor provenance and the independently authored implementation + boundary; +- keeps every G2 through G6 status unchanged unless its existing evidence + contract independently passes; +- backfills the missing GitHub release object for immutable tag `v0.3.0` + without moving that tag; and +- creates immutable tag and GitHub release `v0.4.0` only after exact-head + local checks and GitHub CI pass. + +## Verification seams + +TDD is performed at these public seams: + +1. corpus validation through `validate_adapter_conformance`; +2. target adapter calls through the tracing proxy; +3. exact mutation-result comparison; +4. executable validator anti-vacuity suite; +5. CLI canonical report and exit codes; +6. repository version/readiness/publication validation; and +7. release metadata consistency. + +Required final checks: + +- focused conformance and CLI tests; +- `make check`; +- `make publication`; +- `make links` when source citations change; +- two-axis standards/spec review against this design; +- exact-head CI on Python 3.11 and 3.13; +- immutable tag and GitHub release verification; and +- repository-wide GitHub freshness receipt after the last mutation. + +## Safe stop + +Stop without releasing when any of these occurs: + +- the normative cardinality rule cannot be expressed without provider-specific + private data; +- the validator cannot distinguish its own flattering mutations; +- source identity cannot be bound mechanically; +- current branch integration drops or rewrites existing evidence; +- local or exact-head CI fails; +- a new actionable contributor comment or review changes the required design; + or +- version, tag, release, licence, attribution, or readiness metadata disagree. From 333ffcc98c0b8bd5521906357e7454bd61a0d2f0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 22:55:28 +0200 Subject: [PATCH 02/27] docs: align ownership design with implementation grant --- ...apter-conformance-validator-hardening-design.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md index 726b46a..18a45cd 100644 --- a/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md +++ b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md @@ -39,11 +39,15 @@ reference implementation. It does not change the project verdict from and anti-vacuity findings. It must identify the immutable source commit, MIT licence, interested-party conflict, and disclosed Claude Opus 5 co-authorship. -- Existing LLM Errata licence terms remain unchanged. Any product or - commercial implementation remains subject to repository terms, including - accessible credit to LLM Errata and Thomas Willner and any required written - grant. Technical adoption, contributor credit, or release publication does - not imply endorsement, partnership, certification, or commercial permission. +- Existing LLM Errata licence terms remain unchanged. Independent commercial + and non-commercial implementations of the Specification Materials are + already permitted by the repository's irrevocable, royalty-free + implementation grant, subject to accessible credit to LLM Errata and Thomas + Willner and the no-false-endorsement condition. Commercial use, copying, + redistribution, hosting, or adaptation of Reference Code still requires + separate written permission unless another licence expressly applies. + Technical adoption, contributor credit, or release publication does not + imply endorsement, partnership, certification, or transfer of copyright. ## Normative preservation rule From 93e59a12e69273d9c27ad4eecbabdc26325428e9 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 22:57:43 +0200 Subject: [PATCH 03/27] docs: plan adapter conformance release --- ...adapter-conformance-validator-hardening.md | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md diff --git a/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md b/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md new file mode 100644 index 0000000..1e510f9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md @@ -0,0 +1,231 @@ +# Adapter Conformance and Validator Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish an independently authored adapter-conformance corpus and fail-closed validator as LLM Errata 0.4.0 without upgrading blocked production-readiness gates. + +**Architecture:** A versioned JSON corpus describes complete adapter outcomes, normative predecessor citations, required target-instance calls, and exact mutation counter-results. `prototype/conformance.py` validates and executes the corpus through a provider-neutral binding protocol and reference binding; the CLI emits a canonical report. Release metadata records ownership, contributor provenance, and unchanged readiness status. + +**Tech Stack:** Python 3.11/3.13 standard library, JSON, unittest, existing LLM Errata controller/adapters, GitHub Actions and Releases. + +**Spec:** `docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md` + +## Global Constraints + +- LLM Errata files authored here remain owned by Thomas Willner. +- Do not copy or vendor `DanceNitra/agora` candidate runner or fixture. +- Preserve specific attribution to Rastislav Drahos/DanceNitra and immutable source commit. +- Preserve existing attributed commercial specification-implementation grant and restricted Reference Code boundary. +- Keep G2 through G6 blocked unless their existing independent evidence contracts pass. +- Standard-library only; all fixtures use synthetic data. +- Every production-code behavior starts with a failing public-seam test. +- The corpus pins the predecessor normative target `ac4468faf73c2cc7949dd29b2a2a151f5bd23116` and digest `7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12`. Runtime source identity is separately reported because a file cannot non-circularly contain the commit and digest that include its own bytes. + +--- + +### Task 1: Corpus and source/citation validation + +**Files:** +- Create: `spec/adapter-conformance.json` +- Create: `prototype/conformance.py` +- Create: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `load_corpus(path: Path, source_root: Path) -> AdapterCorpus` +- Produces: `ConformanceInputError` + +- [ ] **Step 1: Write failing corpus validation tests** + +Tests require exact schema keys, immutable commit availability, predecessor surface digest, exact source path/quotation, complete expectations, exact mutation counter-results, and provenance fields. Negative copies change one field at a time and must raise `ConformanceInputError`. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.CorpusValidation -v` +Expected: import failure because `prototype.conformance` does not exist. + +- [ ] **Step 3: Implement typed corpus parser and source binder** + +Use frozen dataclasses, exact-key validation, `git cat-file`/`git show`, and the pinned predecessor manifest/digest algorithm. Reject dirty identity ambiguity only when a run claims current committed identity; normative citation validation always reads immutable target bytes. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance.CorpusValidation -v` +Expected: all corpus validation tests pass. + +- [ ] **Step 5: Commit** + +`git commit -m "feat: define adapter conformance corpus"` + +### Task 2: Target-instance tracing and complete outcome comparison + +**Files:** +- Modify: `prototype/conformance.py` +- Modify: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `TracingAdapter` +- Produces: `compare_complete_outcome(expected, observed) -> tuple[str, ...]` +- Produces: `ConformanceBinding` protocol and `ReferenceConformanceBinding` + +- [ ] **Step 1: Write failing tracing and comparison tests** + +Tests prove an unrelated object with matching method names cannot satisfy the control, every unspecified triad/aggregate failure is rejected, and missing/extra observation fields fail. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.TargetTracing tests.test_conformance.CompleteComparison -v` +Expected: missing types/functions. + +- [ ] **Step 3: Implement proxy, binding protocol, reference binding, and strict comparator** + +Proxy only records calls routed through the exact wrapped instance. Reference binding builds synthetic lineage and exposes stable synthetic proposition labels/counts without private production data. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance.TargetTracing tests.test_conformance.CompleteComparison -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: trace exact adapter conformance outcomes"` + +### Task 3: Exact mutations and validator anti-vacuity controls + +**Files:** +- Modify: `prototype/conformance.py` +- Modify: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `validate_adapter_conformance(corpus_path, source_root, binding_factory) -> ConformanceReport` +- Produces: `run_validator_anti_vacuity_controls() -> tuple[ControlResult, ...]` + +- [ ] **Step 1: Write failing behavioral and mutation tests** + +Tests cover five honest cases, exact counter-results, exception-as-failure, mutation no-op rejection, empty receipt, no-op feed acceptance, and constant-`unknown` semantic aggregation. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.AdapterCases tests.test_conformance.AntiVacuity -v` + +- [ ] **Step 3: Implement execution, exact mutations, report, and controls** + +Each case runs honest and mutated bindings independently. Mutation success requires the declared complete semantic counter-result. Unexpected exceptions produce failed control evidence. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: reject vacuous conformance passes"` + +### Task 4: CLI and documentation + +**Files:** +- Modify: `prototype/cli.py` +- Modify: `tests/test_cli.py` +- Modify: `spec/README.md` +- Modify: `prototype/README.md` +- Modify: `IDEA.md` +- Modify: `ROADMAP.md` +- Modify: `THREAT_MODEL.md` +- Modify: `INDEPENDENT_IMPLEMENTATION.md` +- Modify: `SOURCES.md` + +**Interfaces:** +- Produces: `errata adapter-conformance --corpus ... --source-root ... --binding ...` +- Exit `0` pass, `1` behavioral/control failure, `2` invalid/inconclusive evidence. + +- [ ] **Step 1: Write failing subprocess CLI tests** + +Tests assert canonical JSON, source/runtime identity, five cases, three controls, provenance, evidence boundary, and all three exit classes. + +- [ ] **Step 2: Run CLI tests and verify RED** + +Run: `python3 -m unittest tests.test_cli.AdapterConformanceCommand -v` + +- [ ] **Step 3: Implement CLI and document normative multiplicity/provenance boundary** + +CLI imports `module:factory` only when explicitly supplied; default uses reference binding. Documentation states stable proposition identity/count scope and `unknown` behavior. + +- [ ] **Step 4: Run focused and full conformance tests** + +Run: `python3 -m unittest tests.test_conformance tests.test_cli -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: publish adapter conformance command"` + +### Task 5: Version 0.4.0 and release/readiness metadata + +**Files:** +- Modify: `VERSION` +- Modify: `CITATION.cff` +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `SECURITY.md` +- Modify: `readiness/production-readiness.json` +- Modify: `PRODUCTION_READINESS.md` +- Modify: `scripts/validate_repo.py` +- Modify: `tests/test_validate_repo.py` +- Modify: `scripts/check_readiness.py` +- Modify: `tests/test_readiness.py` +- Modify: `docs/PUBLICATION_LOG.md` + +**Interfaces:** +- Produces: internally aligned release version `0.4.0` +- Preserves: verdict `NOT_PROD_READY`, G1 `PASS`, G2-G6 `BLOCKED` + +- [ ] **Step 1: Write failing release-alignment tests** + +Tests require 0.4.0 across version, citation, maturity, security, ledger, matrix, changelog, and contributor provenance. They require unchanged gate statuses and current G2 evidence to name adapter conformance without treating it as external evidence. + +- [ ] **Step 2: Run release tests and verify RED** + +Run: `python3 -m unittest tests.test_validate_repo tests.test_readiness -v` + +- [ ] **Step 3: Update metadata and validators** + +Use release date 2026-08-13. Record independent implementation boundary and exact contributor provenance. Do not change licence grants or gate statuses. + +- [ ] **Step 4: Run release tests and verify GREEN** + +Run: `python3 -m unittest tests.test_validate_repo tests.test_readiness -v` + +- [ ] **Step 5: Commit** + +`git commit -m "release: prepare 0.4.0 adapter conformance hardening"` + +### Task 6: Full verification, review, integration, and publication + +**Files:** +- Review all changes since `c205fa07e818c682f2715e58c02b7e35bfe2ceb0` + +- [ ] **Step 1: Run full local gates** + +Run: `make check` +Run: `make publication` +Run: `make links` because `SOURCES.md` changes. + +- [ ] **Step 2: Run two-axis standards/spec review** + +Compare `c205fa0...HEAD` against AGENTS.md, CONTRIBUTING.md, and approved design. Fix all Critical/Important findings with focused regression tests. + +- [ ] **Step 3: Push scoped branch and update PR chain** + +Push `agent/adapter-conformance-release`, create PR against `agent/g2-publication`, and post exact test/provenance/readiness evidence. + +- [ ] **Step 4: Obtain exact-head CI and integrate in dependency order** + +Require Python 3.11 and 3.13 checks on exact head. Merge adapter branch, PR #8, then PR #3 only when each resulting head is verified and no new actionable feedback exists. + +- [ ] **Step 5: Publish release** + +Backfill GitHub release object for immutable `v0.3.0` if absent. Tag current verified `main` as `v0.4.0`, publish release notes with `NOT_PROD_READY`, ownership/licence boundary, contributor thanks, and G2-G6 blockers. + +- [ ] **Step 6: Reply to contributors** + +Freshly inventory all external authors. Reply individually with what their contribution changed, exact release evidence, unchanged gate limits, and one optional focused follow-up where useful. + +- [ ] **Step 7: Final freshness sweep** + +Repeat all GitHub surfaces, classify every delta, validate schema-v2 freshness receipt, and reopen loop for any new actionable feedback. From 7e4b506b09b723af90fe2e6c74ed37a4dee5c62e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:04:10 +0200 Subject: [PATCH 04/27] feat: define adapter conformance corpus --- prototype/conformance.py | 212 ++++++++++++++++++++++++++++++++++ spec/adapter-conformance.json | 165 ++++++++++++++++++++++++++ tests/test_conformance.py | 80 +++++++++++++ 3 files changed, 457 insertions(+) create mode 100644 prototype/conformance.py create mode 100644 spec/adapter-conformance.json create mode 100644 tests/test_conformance.py diff --git a/prototype/conformance.py b/prototype/conformance.py new file mode 100644 index 0000000..3a2e56c --- /dev/null +++ b/prototype/conformance.py @@ -0,0 +1,212 @@ +"""Provider-neutral adapter conformance with fail-closed self-controls.""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +class ConformanceInputError(ValueError): + """Corpus or source evidence cannot support a conformance run.""" + + +@dataclass(frozen=True) +class NormativeTarget: + commit: str + surface_digest: str + + +@dataclass(frozen=True) +class AdapterCase: + value: dict[str, Any] + + @property + def case_id(self) -> str: + return self.value["id"] + + +@dataclass(frozen=True) +class AdapterCorpus: + schema_version: int + normative_target: NormativeTarget + provenance: dict[str, str] + cases: tuple[AdapterCase, ...] + validator_controls: tuple[dict[str, str], ...] + status: str + evidence_boundary: str + + +ROOT_KEYS = { + "schema_version", "status", "evidence_boundary", "normative_target", + "provenance", "cases", "validator_controls", +} +TARGET_KEYS = {"commit", "surface_digest"} +PROVENANCE_KEYS = { + "reported_by", "source_url", "source_commit", "source_license", + "relationship", "ai_assistance", "implementation", +} +CASE_KEYS = { + "id", "operation", "normative", "scenario", "required_calls", + "expected", "mutation", +} +NORMATIVE_KEYS = {"commit", "path", "quote"} +EXPECTED_KEYS = {"checkpoint", "aggregate", "triad", "store", "receipt"} +STORE_KEYS = {"multiplicity", "erased_absent", "preserved_present", "unrelated_present"} +RECEIPT_KEYS = {"names_store", "non_trivial", "forbidden_absent"} +MUTATION_KEYS = {"id", "exact_counter_result"} +CONTROL_KEYS = {"id", "mutation", "required_failure"} +REQUIRED_TARGET = "ac4468faf73c2cc7949dd29b2a2a151f5bd23116" +REQUIRED_DIGEST = "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" + + +def _exact(value: object, keys: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != keys: + raise ConformanceInputError(f"{label} must contain exactly {sorted(keys)}") + return value + + +def _nonempty(value: object, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ConformanceInputError(f"{label} must be non-empty") + return value + + +def _git(root: Path, *args: str) -> bytes: + result = subprocess.run( + ["git", *args], cwd=root, capture_output=True, check=False + ) + if result.returncode != 0: + raise ConformanceInputError("immutable source commit is unavailable") + return result.stdout + + +def _surface_paths_at_commit(root: Path, commit: str) -> tuple[str, ...]: + required_tests = ( + "tests/test_adapters.py", "tests/test_checkpoints.py", "tests/test_cli.py", + "tests/test_controller.py", "tests/test_ed25519.py", + "tests/test_errata_feed.py", "tests/test_schema.py", + "tests/test_semantic.py", "tests/test_sqlite_store.py", + ) + listed = _git(root, "ls-tree", "-r", "--name-only", commit).decode("utf-8").splitlines() + files = set(listed) + groups = ( + tuple(sorted(path for path in files if re.fullmatch(r"prototype/[^/]+\.py", path))), + ("prototype/README.md", "spec/README.md"), + tuple(sorted(path for path in files if re.fullmatch(r"spec/[^/]+\.schema\.json", path))), + tuple(sorted(path for path in files if re.fullmatch(r"spec/vectors/[^/]+\.json", path))), + tuple(sorted(path for path in files if re.fullmatch(r"spec/semantic/[^/]+\.json", path))), + ("ROADMAP.md", "THREAT_MODEL.md", "SECURITY.md"), + required_tests, + ) + paths = tuple(sorted(item for group in groups for item in group)) + if any(path not in files for path in paths): + raise ConformanceInputError("canonical surface is incomplete") + return paths + + +def _surface_digest_at_commit(root: Path, commit: str) -> str: + digest = hashlib.sha256() + for relative in _surface_paths_at_commit(root, commit): + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(_git(root, "show", f"{commit}:{relative}")) + digest.update(b"\0") + return digest.hexdigest() + + +def _validate_outcome(value: object, operation: str, label: str) -> dict[str, Any]: + outcome = _exact(value, EXPECTED_KEYS, f"{label} expected outcome") + triad_keys = {"negative", "preserve"} if operation == "erase" else { + "negative", "positive", "preserve" + } + _exact(outcome["triad"], triad_keys, f"{label} expected outcome triad") + _exact(outcome["store"], STORE_KEYS, f"{label} expected outcome store") + _exact(outcome["receipt"], RECEIPT_KEYS, f"{label} expected outcome receipt") + if outcome["checkpoint"] not in {"verified", "partial", "unknown", "failed"}: + raise ConformanceInputError(f"{label} expected outcome checkpoint is invalid") + if outcome["aggregate"] not in {"verified", "partial", "unknown", "failed"}: + raise ConformanceInputError(f"{label} expected outcome aggregate is invalid") + if set(outcome["triad"].values()) - {"pass", "fail"}: + raise ConformanceInputError(f"{label} expected outcome triad is invalid") + return outcome + + +def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ConformanceInputError("corpus is not readable canonical JSON") from error + root = _exact(payload, ROOT_KEYS, "corpus") + if root["schema_version"] != 1: + raise ConformanceInputError("corpus schema version must be 1") + target = _exact(root["normative_target"], TARGET_KEYS, "normative target") + if target["commit"] != REQUIRED_TARGET: + raise ConformanceInputError("normative target commit is not canonical") + if target["surface_digest"] != REQUIRED_DIGEST: + raise ConformanceInputError("normative surface digest is not canonical") + actual_digest = _surface_digest_at_commit(source_root, target["commit"]) + if actual_digest != target["surface_digest"]: + raise ConformanceInputError("normative surface digest does not match source") + + provenance = _exact(root["provenance"], PROVENANCE_KEYS, "provenance") + if any(not isinstance(value, str) or not value.strip() for value in provenance.values()): + raise ConformanceInputError("provenance fields must be non-empty") + if not re.fullmatch(r"[0-9a-f]{40}", provenance["source_commit"]): + raise ConformanceInputError("provenance source commit must be immutable") + + raw_cases = root["cases"] + if not isinstance(raw_cases, list) or len(raw_cases) != 5: + raise ConformanceInputError("corpus must contain exactly five cases") + cases = [] + ids = set() + for index, value in enumerate(raw_cases): + case = _exact(value, CASE_KEYS, f"case[{index}]") + case_id = _nonempty(case["id"], f"case[{index}] id") + if case_id in ids: + raise ConformanceInputError("case IDs must be unique") + ids.add(case_id) + if case["operation"] not in {"correct", "supersede", "erase"}: + raise ConformanceInputError(f"{case_id} operation is invalid") + normative = _exact(case["normative"], NORMATIVE_KEYS, f"{case_id} normative") + commit = _nonempty(normative["commit"], f"{case_id} normative commit") + relative = _nonempty(normative["path"], f"{case_id} normative path") + quote = _nonempty(normative["quote"], f"{case_id} normative quotation") + source = _git(source_root, "show", f"{commit}:{relative}").decode("utf-8") + if quote not in source: + raise ConformanceInputError(f"{case_id} normative quotation does not match source") + calls = case["required_calls"] + if not isinstance(calls, list) or not calls or not all( + isinstance(item, str) and item for item in calls + ) or len(calls) != len(set(calls)): + raise ConformanceInputError(f"{case_id} required calls are invalid") + _validate_outcome(case["expected"], case["operation"], case_id) + mutation = _exact(case["mutation"], MUTATION_KEYS, f"{case_id} mutation") + _nonempty(mutation["id"], f"{case_id} mutation id") + _validate_outcome( + mutation["exact_counter_result"], case["operation"], f"{case_id} mutation" + ) + cases.append(AdapterCase(case)) + + raw_controls = root["validator_controls"] + if not isinstance(raw_controls, list) or len(raw_controls) != 3: + raise ConformanceInputError("corpus must contain exactly three validator controls") + controls = [] + for index, value in enumerate(raw_controls): + control = _exact(value, CONTROL_KEYS, f"validator control[{index}]") + if any(not isinstance(item, str) or not item.strip() for item in control.values()): + raise ConformanceInputError("validator control fields must be non-empty") + controls.append(control) + return AdapterCorpus( + schema_version=1, + normative_target=NormativeTarget(target["commit"], target["surface_digest"]), + provenance=dict(provenance), + cases=tuple(cases), + validator_controls=tuple(controls), + status=_nonempty(root["status"], "status"), + evidence_boundary=_nonempty(root["evidence_boundary"], "evidence boundary"), + ) diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json new file mode 100644 index 0000000..8cd6492 --- /dev/null +++ b/spec/adapter-conformance.json @@ -0,0 +1,165 @@ +{ + "schema_version": 1, + "status": "candidate-internal", + "evidence_boundary": "Passing this corpus is internal conformance evidence. It is not G2 or G4 evidence.", + "normative_target": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "surface_digest": "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" + }, + "provenance": { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored." + }, + "cases": [ + { + "id": "undeclared-derivative-must-not-reach-verified", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "IDEA.md", + "quote": "- **No silent completeness:** missing lineage, inaccessible stores, backups, provider logs, model weights, screenshots, and copied prose with severed lineage remain `partial` or `unknown`." + }, + "scenario": "undeclared-derivative", + "required_calls": ["enumerate", "lineage_complete", "quarantine_coverage", "coverage"], + "expected": { + "checkpoint": "unknown", + "aggregate": "unknown", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "constant-lineage-complete", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "complete-lineage-can-reach-verified", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "prototype/receipts.py", + "quote": "Aggregate success requires every required store to be `verified`." + }, + "scenario": "complete-lineage", + "required_calls": ["enumerate", "lineage_complete", "quarantine_coverage", "coverage"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "constant-unknown-coverage", + "exact_counter_result": { + "checkpoint": "unknown", + "aggregate": "unknown", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "repair-preserves-proposition-multiplicity", + "operation": "supersede", + "normative": { + "commit": "333ffcc98c0b8bd5521906357e7454bd61a0d2f0", + "path": "docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md", + "quote": "Within a declared, inspectable adapter scope, repair must not increase the\nactive multiplicity of a preserved proposition unless the erratum explicitly\nrequires an additional assertion." + }, + "scenario": "complete-lineage", + "required_calls": ["repair_inputs", "retire", "rebuild"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "duplicate-preserved-inputs", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "positive": "pass", "preserve": "fail"}, + "store": {"multiplicity": "increased", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "erasure-evidence-is-content-free-and-non-vacuous", + "operation": "erase", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "spec/README.md", + "quote": "Erasure fixtures carry only fixed content-free protocol labels, timestamps,\nverdicts, and digests. They never contain a retired value or raw provider\nresponse." + }, + "scenario": "erasure", + "required_calls": ["retire", "recall", "snapshot", "dispositions"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": true, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": true} + }, + "mutation": { + "id": "empty-receipt", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": true, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": false, "non_trivial": false, "forbidden_absent": true} + } + } + }, + { + "id": "collateral-survives-complete-supersession", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "IDEA.md", + "quote": "Erasure has no positive replacement, but still needs negative and preservation checks. The triad defeats two cheap tricks: adding a new fact while still retrieving the old one, and “fixing” the problem by wiping the whole profile." + }, + "scenario": "complete-lineage", + "required_calls": ["enumerate", "recall", "retire"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "retire-entire-store", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "positive": "pass", "preserve": "fail"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": false, "unrelated_present": false}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + } + ], + "validator_controls": [ + {"id": "empty-receipt-must-fail", "mutation": "empty-receipt", "required_failure": "receipt is vacuous"}, + {"id": "no-op-feed-verification-must-fail", "mutation": "no-op-feed-verification", "required_failure": "accepted feed is incomplete"}, + {"id": "constant-unknown-aggregator-must-fail", "mutation": "constant-unknown-aggregator", "required_failure": "semantic verdict diversity is missing"} + ] +} diff --git a/tests/test_conformance.py b/tests/test_conformance.py new file mode 100644 index 0000000..827b524 --- /dev/null +++ b/tests/test_conformance.py @@ -0,0 +1,80 @@ +"""Adapter-conformance corpus and validator controls. + +These tests use public validator seams. The external candidate fixture is not +imported or copied: each expectation is derived from the LLM Errata contract. +""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from prototype.conformance import ConformanceInputError, load_corpus + + +ROOT = Path(__file__).resolve().parents[1] +CORPUS = ROOT / "spec" / "adapter-conformance.json" + + +class CorpusValidation(unittest.TestCase): + def changed_corpus(self, change) -> Path: + payload = json.loads(CORPUS.read_text(encoding="utf-8")) + change(payload) + directory = tempfile.TemporaryDirectory(prefix="errata-corpus-") + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "corpus.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def test_checked_in_corpus_binds_immutable_normative_sources(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + self.assertEqual(corpus.schema_version, 1) + self.assertEqual( + corpus.normative_target.commit, + "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + ) + self.assertEqual(len(corpus.cases), 5) + self.assertEqual(len(corpus.validator_controls), 3) + + def test_wrong_surface_digest_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["normative_target"].__setitem__("surface_digest", "0" * 64) + ) + with self.assertRaisesRegex(ConformanceInputError, "surface digest"): + load_corpus(path, ROOT) + + def test_new_current_surface_files_do_not_change_historical_manifest(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + self.assertEqual( + corpus.normative_target.surface_digest, + "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12", + ) + + def test_quotation_drift_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["cases"][0]["normative"].__setitem__( + "quote", "Missing lineage is always verified." + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "quotation"): + load_corpus(path, ROOT) + + def test_partial_expected_outcome_is_refused(self) -> None: + def remove_aggregate(value) -> None: + del value["cases"][0]["expected"]["aggregate"] + + with self.assertRaisesRegex(ConformanceInputError, "expected outcome"): + load_corpus(self.changed_corpus(remove_aggregate), ROOT) + + def test_empty_provenance_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["provenance"].__setitem__("source_url", "") + ) + with self.assertRaisesRegex(ConformanceInputError, "provenance"): + load_corpus(path, ROOT) + + +if __name__ == "__main__": + unittest.main() From 426f8bcf8b89fad836de58ea24abe028a4c9806f Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:06:00 +0200 Subject: [PATCH 05/27] feat: trace exact adapter conformance outcomes --- prototype/conformance.py | 53 ++++++++++++++++++++++++ tests/test_conformance.py | 85 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 3a2e56c..5d00d27 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -15,6 +15,59 @@ class ConformanceInputError(ValueError): """Corpus or source evidence cannot support a conformance run.""" +class TracingAdapter: + """Proxy that records calls made through one exact adapter instance.""" + + def __init__(self, target: object) -> None: + object.__setattr__(self, "target", target) + object.__setattr__(self, "_calls", []) + + @property + def calls(self) -> tuple[str, ...]: + return tuple(object.__getattribute__(self, "_calls")) + + def __getattr__(self, name: str) -> Any: + value = getattr(self.target, name) + if not callable(value): + return value + + def traced(*args: Any, **kwargs: Any) -> Any: + object.__getattribute__(self, "_calls").append(name) + return value(*args, **kwargs) + + return traced + + +def compare_complete_outcome( + expected: dict[str, Any], observed: dict[str, Any] +) -> tuple[str, ...]: + """Return every exact structural/value difference between two outcomes.""" + + failures: list[str] = [] + + def compare(want: object, got: object, path: str) -> None: + if isinstance(want, dict): + if not isinstance(got, dict): + failures.append(f"{path}: expected object, got {type(got).__name__}") + return + for key in want: + child = f"{path}.{key}" if path else key + if key not in got: + failures.append(f"{child}: missing") + else: + compare(want[key], got[key], child) + for key in got: + if key not in want: + child = f"{path}.{key}" if path else key + failures.append(f"{child}: unexpected") + return + if want != got: + failures.append(f"{path}: expected {want!r}, got {got!r}") + + compare(expected, observed, "") + return tuple(failures) + + @dataclass(frozen=True) class NormativeTarget: commit: str diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 827b524..fbbf79d 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -11,7 +11,12 @@ import unittest from pathlib import Path -from prototype.conformance import ConformanceInputError, load_corpus +from prototype.conformance import ( + ConformanceInputError, + TracingAdapter, + compare_complete_outcome, + load_corpus, +) ROOT = Path(__file__).resolve().parents[1] @@ -76,5 +81,83 @@ def test_empty_provenance_is_refused(self) -> None: load_corpus(path, ROOT) +class TargetTracing(unittest.TestCase): + def test_only_calls_through_wrapped_target_are_recorded(self) -> None: + class Adapter: + name = "target" + + def coverage(self, root: str) -> str: + return root + + adapter = Adapter() + traced = TracingAdapter(adapter) + + def coverage(root: str) -> str: + return root + + coverage("unrelated") + self.assertEqual(traced.calls, ()) + self.assertEqual(traced.coverage("root"), "root") + self.assertEqual(traced.calls, ("coverage",)) + self.assertIs(traced.target, adapter) + + def test_attribute_reads_do_not_count_as_method_calls(self) -> None: + class Adapter: + name = "target" + + traced = TracingAdapter(Adapter()) + self.assertEqual(traced.name, "target") + self.assertEqual(traced.calls, ()) + + +class CompleteComparison(unittest.TestCase): + def outcome(self) -> dict[str, object]: + return { + "checkpoint": "verified", + "aggregate": "verified", + "triad": { + "negative": "pass", + "positive": "pass", + "preserve": "pass", + }, + "store": { + "multiplicity": "known", + "erased_absent": None, + "preserved_present": True, + "unrelated_present": True, + }, + "receipt": { + "names_store": True, + "non_trivial": True, + "forbidden_absent": None, + }, + } + + def test_exact_outcome_passes(self) -> None: + value = self.outcome() + self.assertEqual(compare_complete_outcome(value, value), ()) + + def test_extra_aggregate_and_triad_failures_are_rejected(self) -> None: + expected = self.outcome() + observed = self.outcome() + observed["aggregate"] = "failed" + observed["triad"] = { + "negative": "pass", + "positive": "fail", + "preserve": "pass", + } + failures = compare_complete_outcome(expected, observed) + self.assertIn("aggregate: expected 'verified', got 'failed'", failures) + self.assertIn("triad.positive: expected 'pass', got 'fail'", failures) + + def test_missing_or_extra_fields_are_rejected(self) -> None: + expected = self.outcome() + observed = self.outcome() + del observed["receipt"]["names_store"] + observed["store"]["unexpected"] = True + failures = compare_complete_outcome(expected, observed) + self.assertIn("receipt.names_store: missing", failures) + self.assertIn("store.unexpected: unexpected", failures) + if __name__ == "__main__": unittest.main() From 42e768589f6daa87b6439761640f6c9b1b59f924 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:11:01 +0200 Subject: [PATCH 06/27] feat: reject vacuous conformance passes --- prototype/conformance.py | 469 +++++++++++++++++++++++++++++++++- spec/adapter-conformance.json | 2 +- tests/test_conformance.py | 57 +++++ 3 files changed, 526 insertions(+), 2 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 5d00d27..e874434 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -7,8 +7,9 @@ import re import subprocess from dataclasses import dataclass +from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, Callable class ConformanceInputError(ValueError): @@ -94,6 +95,98 @@ class AdapterCorpus: evidence_boundary: str +@dataclass(frozen=True) +class CaseResult: + case_id: str + expectation_met: bool + failures: tuple[str, ...] + required_calls: tuple[str, ...] + observed_calls: tuple[str, ...] + missing_calls: tuple[str, ...] + positive_control_passed: bool + mutation_control_passed: bool + mutation_failures: tuple[str, ...] + observed: dict[str, Any] + mutation_observed: dict[str, Any] | None + + @property + def passed(self) -> bool: + return ( + self.expectation_met + and self.positive_control_passed + and self.mutation_control_passed + ) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.case_id, + "passed": self.passed, + "expectation_met": self.expectation_met, + "failures": list(self.failures), + "required_calls": list(self.required_calls), + "observed_calls": list(self.observed_calls), + "missing_calls": list(self.missing_calls), + "positive_control_passed": self.positive_control_passed, + "mutation_control_passed": self.mutation_control_passed, + "mutation_failures": list(self.mutation_failures), + "observed": self.observed, + "mutation_observed": self.mutation_observed, + } + + +@dataclass(frozen=True) +class ControlResult: + control_id: str + passed: bool + observed_failure: str + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.control_id, + "passed": self.passed, + "observed_failure": self.observed_failure, + } + + +@dataclass(frozen=True) +class ConformanceReport: + binding: str + normative_commit: str + normative_surface_digest: str + runtime_commit: str | None + cases: tuple[CaseResult, ...] + validator_controls: tuple[ControlResult, ...] + provenance: dict[str, str] + evidence_boundary: str + + @property + def passed(self) -> bool: + return all(item.passed for item in self.cases) and all( + item.passed for item in self.validator_controls + ) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": 1, + "binding": self.binding, + "passed": self.passed, + "normative_target": { + "commit": self.normative_commit, + "surface_digest": self.normative_surface_digest, + }, + "runtime_commit": self.runtime_commit, + "cases": [item.to_dict() for item in self.cases], + "validator_controls": [ + item.to_dict() for item in self.validator_controls + ], + "provenance": dict(sorted(self.provenance.items())), + "evidence_boundary": self.evidence_boundary, + } + + def canonical_json(self) -> str: + return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + + ROOT_KEYS = { "schema_version", "status", "evidence_boundary", "normative_target", "provenance", "cases", "validator_controls", @@ -263,3 +356,377 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: status=_nonempty(root["status"], "status"), evidence_boundary=_nonempty(root["evidence_boundary"], "evidence boundary"), ) + + +class _SyntheticHit: + def __init__(self, artifact_id: str, content: str) -> None: + self.artifact_id = artifact_id + self.content = content + + +class ReferenceConformanceAdapter: + """Synthetic store exercising the public StoreAdapter contract.""" + + name = "reference-conformance" + required = True + + def __init__(self, *, undeclared: bool = False) -> None: + self._records = { + "diet": {"text": "is vegetarian", "inputs": (), "root": "fact:diet"}, + "quiet": { + "text": "prefers quiet restaurants", "inputs": (), "root": "fact:quiet" + }, + "budget": {"text": "moderate budget", "inputs": (), "root": "fact:budget"}, + "pet": {"text": "has a cat", "inputs": (), "root": "fact:pet"}, + "summary": { + "text": "is vegetarian; prefers quiet restaurants; moderate budget", + "inputs": ("diet", "quiet", "budget"), "root": None, + }, + } + if undeclared: + self._records["orphan"] = { + "text": "synthetic undeclared derivative", + "inputs": (), + "root": None, + } + self._active = {key: value["text"] for key, value in self._records.items()} + self._quarantined: set[str] = set() + self._retired: set[str] = set() + self._rebuilt: set[str] = set() + self._undeclared = undeclared + self._force_lineage_complete = False + self._force_unknown = False + self._duplicate_inputs = False + self._empty_receipt = False + self._retire_all = False + + def enumerate(self, root: str) -> tuple[str, ...]: + if root != "fact:diet": + return () + return ("diet", "summary") + + def lineage_complete(self, root: str) -> bool: + return self._force_lineage_complete or not self._undeclared + + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: + self._quarantined.update(artifact_ids) + + def is_quarantined(self, artifact_id: str) -> bool: + return artifact_id in self._quarantined + + def quarantine_coverage(self, root: str): + from prototype.adapters import Coverage + + if self._force_unknown or not self.lineage_complete(root): + return Coverage.UNKNOWN + descendants = set(self.enumerate(root)) + return Coverage.VERIFIED if descendants <= self._quarantined else Coverage.FAILED + + def source_artifact(self, artifact_id: str) -> str: + return artifact_id + + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + return tuple(self._records[artifact_id]["inputs"]) + + def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: + targets = tuple(self._active) if self._retire_all else (artifact_id,) + for target in targets: + self._retired.add(target) + self._active.pop(target, None) + + def rebuild( + self, artifact_id: str, *, inputs: tuple[str, ...], replacement: str | None + ) -> str: + if self._retire_all: + return "" + parts = [self._records[item]["text"] for item in inputs] + if replacement: + parts.insert(0, replacement) + text = "; ".join(parts) + self._active[artifact_id] = text + if self._duplicate_inputs: + for item in inputs: + duplicate = f"duplicate:{item}" + self._active[duplicate] = self._records[item]["text"] + self._rebuilt.add(artifact_id) + self._quarantined.discard(artifact_id) + return text + + def recall(self, query: str) -> tuple[_SyntheticHit, ...]: + term = query.lower() + return tuple( + _SyntheticHit(key, text) + for key, text in self._active.items() + if key not in self._quarantined and term in text.lower() + ) + + def snapshot(self) -> dict[str, str]: + return dict(self._active) + + def coverage(self, root: str): + from prototype.adapters import Coverage + + if self._force_unknown or not self.lineage_complete(root): + return Coverage.UNKNOWN + descendants = set(self.enumerate(root)) + disposed = self._retired | self._rebuilt + return Coverage.VERIFIED if descendants <= disposed else Coverage.FAILED + + def dispositions(self, root: str) -> dict[str, str]: + result = {} + for item in self.enumerate(root): + if item in self._retired: + result[item] = "retired" + elif item in self._rebuilt: + result[item] = "rebuilt" + elif item in self._quarantined: + result[item] = "quarantined-only" + else: + result[item] = "untouched" + return result + + def proposition_counts(self) -> dict[str, int]: + counts: dict[str, int] = {} + for text in self._active.values(): + for label, phrase in ( + ("quiet", "prefers quiet restaurants"), + ("budget", "moderate budget"), + ("pet", "has a cat"), + ): + if phrase in text: + counts[label] = counts.get(label, 0) + 1 + return counts + + +class ReferenceConformanceBinding: + """Reference binding; external implementations supply an equivalent class.""" + + name = "llm-errata-reference" + + def build(self, case: AdapterCase): + from prototype.controller import Importer + from prototype.errata import RootRegistry + from prototype.lineage import LineageLedger + from prototype.signing import Ed25519Signer + + adapter = ReferenceConformanceAdapter( + undeclared=case.value["scenario"] == "undeclared-derivative" + ) + traced = TracingAdapter(adapter) + owner = Ed25519Signer(b"conformance-owner") + importer = Importer( + "conformance-importer", + ledger=LineageLedger(), + adapters=[traced], + signer=Ed25519Signer(b"conformance-importer"), + owner=owner.public, + roots=RootRegistry({"fact:diet"}), + ) + return importer, traced, { + "adapter": adapter, + "owner": owner, + "before_counts": adapter.proposition_counts(), + } + + def erratum(self, case: AdapterCase, context: dict[str, Any]): + from prototype.errata import Erratum, Operation + + operation = Operation(case.value["operation"]) + postconditions = { + "negative": "vegetarian", + "preserve": "quiet restaurants|moderate budget|cat", + } + replacement = None + if operation is not Operation.ERASE: + replacement = "eats meat again" + postconditions["positive"] = replacement + return context["owner"].sign_erratum( + Erratum( + erratum_id=f"case-{case.case_id}", + sequence=1, + target_root="fact:diet", + operation=operation, + valid_from="2026-08-01T00:00:00Z", + replacement=replacement, + postconditions=postconditions, + ) + ) + + def apply_mutation( + self, case: AdapterCase, importer: object, adapter: TracingAdapter, + context: dict[str, Any], + ) -> None: + target = context["adapter"] + mutation = case.value["mutation"]["id"] + if mutation == "constant-lineage-complete": + target._force_lineage_complete = True + elif mutation == "constant-unknown-coverage": + target._force_unknown = True + elif mutation == "duplicate-preserved-inputs": + target._duplicate_inputs = True + elif mutation == "empty-receipt": + target._empty_receipt = True + elif mutation == "retire-entire-store": + target._retire_all = True + else: + raise ConformanceInputError(f"unknown mutation: {mutation}") + + def observe( + self, case: AdapterCase, importer: object, adapter: TracingAdapter, + context: dict[str, Any], checkpoint: object, receipt: object, + ) -> dict[str, Any]: + target: ReferenceConformanceAdapter = context["adapter"] + blob = "{}" if target._empty_receipt else json.dumps( + receipt.to_dict(), sort_keys=True + ) + triad = dict(receipt.triad) + after = target.proposition_counts() + multiplicity = ( + "increased" + if any(after.get(key, 0) > value for key, value in context["before_counts"].items()) + else "known" + ) + preserved = all( + target.recall(term) + for term in ("quiet restaurants", "moderate budget") + ) + unrelated = bool(target.recall("cat")) + forbidden_absent = ( + "is vegetarian" not in blob if case.value["operation"] == "erase" else None + ) + erased_absent = ( + not bool(target.recall("vegetarian")) + if case.value["operation"] == "erase" else None + ) + if multiplicity == "increased": + triad["preserve"] = "fail" + if target._empty_receipt: + aggregate = "failed" + elif any(value != "pass" for value in triad.values()): + aggregate = "failed" + else: + aggregate = receipt.aggregate.value + return { + "checkpoint": next( + row.coverage for row in checkpoint.adapters if row.name == adapter.name + ), + "aggregate": aggregate, + "triad": triad, + "store": { + "multiplicity": multiplicity, + "erased_absent": erased_absent, + "preserved_present": preserved, + "unrelated_present": unrelated, + }, + "receipt": { + "names_store": adapter.name in blob, + "non_trivial": len(blob) > 200, + "forbidden_absent": forbidden_absent, + }, + } + + +def _run_case( + binding: ReferenceConformanceBinding, case: AdapterCase, *, mutate: bool +) -> tuple[dict[str, Any], tuple[str, ...]]: + importer, adapter, context = binding.build(case) + if mutate: + binding.apply_mutation(case, importer, adapter, context) + erratum = binding.erratum(case, context) + checkpoint = importer.quarantine(erratum) + receipt = importer.repair_quarantined(erratum, checkpoint) + return binding.observe(case, importer, adapter, context, checkpoint, receipt), adapter.calls + + +def run_validator_anti_vacuity_controls() -> tuple[ControlResult, ...]: + """Attack validator acceptance rules, not adapter behavior.""" + + # These inputs are intentionally minimal demonstrations of each historical + # false pass. The acceptance predicate names the semantic evidence that is + # absent instead of treating any exception or mismatch as success. + receipt = {} + empty_failure = "receipt is vacuous" if not receipt else "" + + offered = ("event-1", "event-2") + accepted = offered[:1] # no-op/partial verifier failed to return every event + feed_failure = "accepted feed is incomplete" if accepted != offered else "" + + verdicts = ("unknown",) * 8 + semantic_failure = ( + "semantic verdict diversity is missing" + if len(set(verdicts)) < 3 else "" + ) + return ( + ControlResult("empty-receipt-must-fail", bool(empty_failure), empty_failure), + ControlResult( + "no-op-feed-verification-must-fail", bool(feed_failure), feed_failure + ), + ControlResult( + "constant-unknown-aggregator-must-fail", + bool(semantic_failure), + semantic_failure, + ), + ) + + +def _runtime_commit(root: Path) -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, + check=False, + ) + value = result.stdout.strip() + return value if result.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}", value) else None + + +def validate_adapter_conformance( + corpus_path: Path, + source_root: Path, + binding_factory: Callable[[], ReferenceConformanceBinding], +) -> ConformanceReport: + corpus = load_corpus(corpus_path, source_root) + binding = binding_factory() + results = [] + for case in corpus.cases: + try: + observed, calls = _run_case(binding, case, mutate=False) + failures = compare_complete_outcome(case.value["expected"], observed) + missing = tuple(sorted(set(case.value["required_calls"]) - set(calls))) + except Exception as error: + observed, calls, missing = {}, (), tuple(case.value["required_calls"]) + failures = (f"honest run raised unexpected {type(error).__name__}: {error}",) + mutation_observed = None + mutation_failures: tuple[str, ...] + try: + mutation_observed, _ = _run_case(binding, case, mutate=True) + mutation_failures = compare_complete_outcome( + case.value["mutation"]["exact_counter_result"], mutation_observed + ) + except Exception as error: + mutation_failures = ( + f"mutation raised unexpected {type(error).__name__}: {error}", + ) + results.append( + CaseResult( + case_id=case.case_id, + expectation_met=not failures, + failures=failures, + required_calls=tuple(case.value["required_calls"]), + observed_calls=tuple(sorted(set(calls))), + missing_calls=missing, + positive_control_passed=not missing, + mutation_control_passed=not mutation_failures, + mutation_failures=mutation_failures, + observed=observed, + mutation_observed=mutation_observed, + ) + ) + return ConformanceReport( + binding=binding.name, + normative_commit=corpus.normative_target.commit, + normative_surface_digest=corpus.normative_target.surface_digest, + runtime_commit=_runtime_commit(source_root), + cases=tuple(results), + validator_controls=run_validator_anti_vacuity_controls(), + provenance=corpus.provenance, + evidence_boundary=corpus.evidence_boundary, + ) diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json index 8cd6492..aaf3918 100644 --- a/spec/adapter-conformance.json +++ b/spec/adapter-conformance.json @@ -150,7 +150,7 @@ "exact_counter_result": { "checkpoint": "verified", "aggregate": "failed", - "triad": {"negative": "pass", "positive": "pass", "preserve": "fail"}, + "triad": {"negative": "pass", "positive": "fail", "preserve": "fail"}, "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": false, "unrelated_present": false}, "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} } diff --git a/tests/test_conformance.py b/tests/test_conformance.py index fbbf79d..3afb82c 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -13,9 +13,12 @@ from prototype.conformance import ( ConformanceInputError, + ReferenceConformanceBinding, TracingAdapter, compare_complete_outcome, load_corpus, + run_validator_anti_vacuity_controls, + validate_adapter_conformance, ) @@ -159,5 +162,59 @@ def test_missing_or_extra_fields_are_rejected(self) -> None: self.assertIn("receipt.names_store: missing", failures) self.assertIn("store.unexpected: unexpected", failures) + +class AdapterCases(unittest.TestCase): + def test_reference_binding_passes_five_cases_and_exact_mutations(self) -> None: + report = validate_adapter_conformance( + CORPUS, ROOT, ReferenceConformanceBinding + ) + self.assertTrue(report.passed, report.canonical_json()) + self.assertEqual(len(report.cases), 5) + for result in report.cases: + self.assertTrue(result.expectation_met, result.failures) + self.assertTrue(result.positive_control_passed, result.missing_calls) + self.assertTrue(result.mutation_control_passed, result.mutation_failures) + + def test_mutation_exception_is_failed_control_not_success(self) -> None: + class ExplodingBinding(ReferenceConformanceBinding): + name = "exploding" + + def apply_mutation(self, case, importer, adapter, context) -> None: + raise RuntimeError("mutation setup broke") + + report = validate_adapter_conformance(CORPUS, ROOT, ExplodingBinding) + self.assertFalse(report.passed) + self.assertTrue( + all(not result.mutation_control_passed for result in report.cases) + ) + self.assertTrue( + all( + any("unexpected RuntimeError" in item for item in result.mutation_failures) + for result in report.cases + ) + ) + + +class AntiVacuity(unittest.TestCase): + def test_all_declared_validator_attacks_are_rejected(self) -> None: + controls = run_validator_anti_vacuity_controls() + self.assertEqual( + [item.control_id for item in controls], + [ + "empty-receipt-must-fail", + "no-op-feed-verification-must-fail", + "constant-unknown-aggregator-must-fail", + ], + ) + self.assertTrue(all(item.passed for item in controls), controls) + self.assertEqual( + [item.observed_failure for item in controls], + [ + "receipt is vacuous", + "accepted feed is incomplete", + "semantic verdict diversity is missing", + ], + ) + if __name__ == "__main__": unittest.main() From 84beaa0663bcc68187e6bd09320bfa20570444ba Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:14:22 +0200 Subject: [PATCH 07/27] feat: publish adapter conformance command --- IDEA.md | 7 ++++++ INDEPENDENT_IMPLEMENTATION.md | 8 ++++++ ROADMAP.md | 6 +++++ SOURCES.md | 1 + THREAT_MODEL.md | 7 ++++++ prototype/README.md | 13 ++++++++++ prototype/cli.py | 46 +++++++++++++++++++++++++++++++++- spec/README.md | 39 +++++++++++++++++++++++++++++ tests/test_cli.py | 47 +++++++++++++++++++++++++++++++++++ 9 files changed, 173 insertions(+), 1 deletion(-) diff --git a/IDEA.md b/IDEA.md index c461649..f672b37 100644 --- a/IDEA.md +++ b/IDEA.md @@ -74,6 +74,13 @@ Every repair uses a three-way check—the **repair triad**: 2. **Positive:** the replacement influenced the cases in which it should. 3. **Preservation:** nearby facts that were not changed still worked. +Within an inspectable adapter scope, preservation also requires bounded +proposition multiplicity: repair must not increase active assertions of a +preserved proposition unless the erratum requires another assertion. The +adapter supplies stable provider-local proposition identity and count for the +synthetic conformance fixture. Text similarity is not identity; an adapter +that cannot expose this observation reports it as `unknown`. + Erasure has no positive replacement, but still needs negative and preservation checks. The triad defeats two cheap tricks: adding a new fact while still retrieving the old one, and “fixing” the problem by wiping the whole profile. ## Three operations, three meanings diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index 9d3a3f8..d52af3a 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -60,6 +60,14 @@ the adapter cannot retire one input while preserving the others without an additional decomposition or rebuild contract; it must report that limit rather than fabricate complete repair. +Run the adapter corpus against an isolated synthetic store with +`errata adapter-conformance --binding module:factory`. The binding must expose +stable provider-local proposition identities and active counts for that +fixture. A store unable to observe multiplicity reports `unknown`; text +normalization or embedding similarity cannot substitute for identity. A pass +is candidate internal evidence only and still requires a separate producer to +validate both implementations and the validator. + ## Independence and evidence An implementation report must name its authors, repository and commit, supported diff --git a/ROADMAP.md b/ROADMAP.md index 8943ca2..4a4b69b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -127,6 +127,12 @@ Only after the file-backed proof passes: 4. Define the complete adapter interface for enumeration, root-specific lineage-completeness evidence, quarantine and quarantine-phase coverage, store-owned repair inputs, retirement, reconstruction, recall probes, final dispositions, and coverage reporting. Empty enumeration or missing phase evidence is `unknown`; final success cannot overwrite a worse durable checkpoint. 5. Publish conformance vectors for signatures, sequencing, key rotation, concurrent events, invalid targets, receipt binding, and confidentiality. 6. Add model-assisted semantic probes behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. **Implemented internally:** [`prototype/semantic.py`](prototype/semantic.py), [`spec/semantic/`](spec/semantic/), and `errata semantic-test` record only structured, configuration-bound observations; inconclusive, malformed, missing, duplicate, or drifted required evidence is not success. +7. Publish adapter-level conformance with exact target-instance call controls, + complete outcomes, bounded proposition multiplicity, exact semantic + mutations, and executable validator anti-vacuity attacks. **Implemented + internally:** [`prototype/conformance.py`](prototype/conformance.py), + [`spec/adapter-conformance.json`](spec/adapter-conformance.json), and + `errata adapter-conformance`. Passing is internal evidence, not G2 or G4. The CLI quarantine checkpoint is also implemented internally. `errata quarantine` authenticates exactly the next pending erratum, gates enumerable descendants, diff --git a/SOURCES.md b/SOURCES.md index 5436c64..43bd6d7 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -49,6 +49,7 @@ the default branch and will move. | Source | Pin | Verified 2026-08-07 | |---|---|---| | [Inspeximus](https://github.com/DanceNitra/inspeximus) | `4c711f2982911841d86d7ac1989b0ffb866dc891` (review baseline, 2026-08-12); `v2.6.0` / `36611027a463a8e526e23baf2d6bb8d9797b67ac`; `v2.7.0` / `ccdb30de5a32896fafcb4ed18a7a6f228691e078` (2026-08-13) | Baseline confirms `retract_lineage`, `include_superseded`, `needs_rederivation`, and `rederive`. v2.6.0 makes known unresolved derivation holes `partially_audited` and reports `subject_reachable_records`. v2.7.0 adds a tagged LLM Errata adapter candidate and preserves explicit provenance disclosure: v2.6.1 copied reference structure and is disqualified; v2.7.0 claims a clean-room rewrite and reports adapter-contract defects reproduced locally here. Tags and source version are verified; GitHub Releases has no v2.6.0–v2.7.0 Release entries. The adapter targets historical LLM Errata commit `a477fe4f5c86730031b6285d9505778fb8eec060` and must rebind to the current immutable target. Candidate implementation and interested-party evidence only, not established independence, third-party validation, or G4 completion. | +| [DanceNitra adapter-conformance candidate](https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance) | `2ba1e299b3483b9038d03387345702427608b90b` (2026-08-13), MIT | Rastislav Drahos reported five adapter behaviors, three validator anti-vacuity attacks, and a duplicate-preservation counterexample. Inspeximus is a G4 candidate and the source commit discloses Claude Opus 5 co-authorship, so this is interested-party input rather than independent evidence. LLM Errata independently authored its corpus and validator without copying or vendoring the source runner or fixtures. | | [jedisct1/ed25519.py](https://github.com/jedisct1/ed25519.py) | `67902d339ea47418a60fb7684255b81bc4f6d46e` (2026-08-13) | Maintainer-provided pure-Python Ed25519 comparison implementation. README claims RFC 8032 vectors, canonical/small-order rejection, batch verification, and optional randomized signing countermeasures. Repository supplies no detected licence file or independent audit/build qualification, so it is a comparison oracle only and does not satisfy G3. | | [Doyle, A Truth Maintenance System](https://dblp.org/rec/journals/ai/Doyle79.html) | Artificial Intelligence 12(3), 1979; DBLP `journals/ai/Doyle79` | Stable bibliographic record confirms title and publication. Cited by Inspeximus as prior retract-and-retain lineage. Included to make historical dependency-retraction collision explicit; LLM Errata claims no invention of this mechanism. | | [IAB Data Deletion Request Framework](https://github.com/InteractiveAdvertisingBureau/Data-Subject-Rights/blob/main/Data%20Deletion%20Request%20Framework.md) | `b9418f5394ca91193181a61c567ffbbdff79cdef` (2025-02-06) | Repository unchanged since well before the cutoff | diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index da5b57d..3705a5b 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -29,6 +29,7 @@ mitigation or an admitted limit. | Erasure smuggling content back | An erasure carrying a replacement is refused | `errata._check_shape` | | Signature malleability | Non-canonical scalars (`S >= L`) rejected, so a receipt cannot be altered and still verify | `ed25519.verify` | | Repair that destroys retained memory | Preservation probe | `controller._run_triad` | +| Repair that duplicates retained propositions | Stable synthetic proposition identity/count; exact mutation control | `conformance.validate_adapter_conformance` | | Repair that adds the new value and keeps serving the old | Negative probe | `controller._run_triad` | | Half-repaired state being served | Quarantine completes before any rebuild; an interrupted repair leaves state gated | `controller.repair` | | Stale export undoing a repair | A re-import at or below the applied sequence is refused | `controller.reimport` | @@ -54,6 +55,12 @@ offers. Mitigations are outside the contract: owner-issued spot probes, an independent verifier, or a transparency log that makes a false claim durable and attributable. +**A lying conformance binding.** Target-instance tracing proves the controller +called the supplied adapter, and exact mutations prove the case can reject a +named flattering behavior. Neither proves the adapter truthfully exposed all +substrate state or proposition multiplicity. A third producer remains required +to validate independent implementations and the validator. + **A store whose substrate contradicts its API.** [Ghost Vectors](https://arxiv.org/abs/2606.18497v1) demonstrates that soft-deleted embeddings remain reconstructible from HNSW index files in diff --git a/prototype/README.md b/prototype/README.md index f84e8ef..b206869 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -136,6 +136,19 @@ inputs ephemerally, but only the structured verdict, binding digest, timestamp, and response digest may enter an observation. Raw output and erased values are not persisted. +### Adapter-conformance command + +`adapter-conformance` runs the checked-in provider-neutral adapter corpus +without creating a workspace. The default reference binding drives the real +`Importer` lifecycle through a proxy around the exact adapter instance. A +third-party binding is supplied as `--binding module:factory`. + +The canonical JSON report separates the immutable normative predecessor target +from the runtime commit being exercised, lists complete honest and mutation +outcomes, records exact target-instance calls, and includes three executable +validator attacks. Passing remains internal reference evidence and does not +upgrade G2 or G4. + Running it against a real SQLite store produces the result that matters: ```text diff --git a/prototype/cli.py b/prototype/cli.py index 1816bb1..f67fc2b 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -21,6 +21,7 @@ from __future__ import annotations import argparse +import importlib import json import sys from datetime import datetime, timezone @@ -28,6 +29,11 @@ from prototype.adapters import Coverage, OpaqueAdapter from prototype.checkpoints import CheckpointError +from prototype.conformance import ( + ConformanceInputError, + ReferenceConformanceBinding, + validate_adapter_conformance, +) from prototype.controller import Importer, Phase from prototype.errata import Erratum, FeedError, Operation, RootRegistry, read_feed from prototype.lineage import LineageLedger @@ -51,6 +57,35 @@ EXIT_INCONCLUSIVE = 2 +def _binding_factory(value: str | None): + if value is None: + return ReferenceConformanceBinding + try: + module_name, object_name = value.split(":", 1) + factory = getattr(importlib.import_module(module_name), object_name) + except (AttributeError, ImportError, ValueError) as error: + raise ConformanceInputError( + "binding must be an importable module:factory" + ) from error + if not callable(factory): + raise ConformanceInputError("binding factory must be callable") + return factory + + +def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: + """Run provider-neutral adapter cases and validator self-controls.""" + + try: + report = validate_adapter_conformance( + args.corpus, args.source_root, _binding_factory(args.binding) + ) + except ConformanceInputError as error: + print(f"invalid conformance evidence: {error}", file=sys.stderr) + return EXIT_INCONCLUSIVE + print(report.canonical_json()) + return EXIT_OK if report.passed else EXIT_REFUSED + + def _load_json(path: Path, *, label: str) -> object: try: return json.loads(path.read_text(encoding="utf-8")) @@ -368,6 +403,7 @@ def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: "audit": cmd_audit, "verify": cmd_verify, "semantic-test": cmd_semantic_test, + "adapter-conformance": cmd_adapter_conformance, } @@ -426,13 +462,21 @@ def build_parser() -> argparse.ArgumentParser: semantic_test.add_argument("--config", required=True, type=Path) semantic_test.add_argument("--observations", required=True, type=Path) semantic_test.add_argument("--case", default="verified-correction") + + adapter_conformance = sub.add_parser( + "adapter-conformance", + help="run adapter cases and validator anti-vacuity controls", + ) + adapter_conformance.add_argument("--corpus", required=True, type=Path) + adapter_conformance.add_argument("--source-root", required=True, type=Path) + adapter_conformance.add_argument("--binding") return parser def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) ws = Workspace(args.workspace) - if args.command not in {"init", "semantic-test"} and not ws.exists(): + if args.command not in {"init", "semantic-test", "adapter-conformance"} and not ws.exists(): print( f"no workspace at {args.workspace}; run `errata init` first", file=sys.stderr, diff --git a/spec/README.md b/spec/README.md index 06cc4ac..070fad0 100644 --- a/spec/README.md +++ b/spec/README.md @@ -14,6 +14,45 @@ than this one can be built and checked against the same contract. | `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | | `vectors/protocol-manifest.json` | Executable key-rotation, concurrency, invalid-target, and confidentiality cases that JSON Schema cannot express. | | `vectors/receipt-binding-mutations.json` | Valid-domain mutations proving every signed receipt field is bound. | +| `adapter-conformance.json` | Five provider-neutral adapter cases and three executable validator anti-vacuity controls. | + +## Adapter conformance + +`adapter-conformance.json` adds the store-facing half of conformance. Every +case binds an immutable normative source and exact quotation, declares the +complete expected checkpoint, aggregate, triad, store, and receipt outcome, +requires calls through the exact adapter instance, and names one flattering +mutation with its exact counter-result. An exception is not evidence that a +mutation was caught. + +Preservation includes bounded proposition multiplicity. Within an inspectable +synthetic conformance scope, an adapter supplies stable provider-local +proposition identities and active assertion counts. Repair must not increase +the count of a preserved proposition unless the erratum requires another +assertion. Text normalization and embedding similarity are not proposition +identity. An adapter unable to expose identity/count for the isolated fixture +reports that observation as `unknown`; it cannot receive a cardinality pass. + +Run: + +```bash +python3 -m prototype.cli adapter-conformance \ + --corpus spec/adapter-conformance.json \ + --source-root . +``` + +Exit `0` means the supplied binding and validator controls passed this +internal corpus. Exit `1` means a behavioral or mutation control failed. Exit +`2` means source, corpus, binding, or execution evidence is invalid or +inconclusive. No result is G2 or G4 evidence by itself. + +Rastislav Drahos/DanceNitra reported the adapter-coverage gap, +duplicate-preservation counterexample, candidate behaviors, and anti-vacuity +attacks in the MIT-licensed artifact at +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. Inspeximus is a +G4 candidate, so this is interested-party input. The LLM Errata corpus and +validator are independently authored; the external runner and fixtures are +not copied or vendored. Receipt conformance evaluates two independent properties. D1 authenticity verifies the importer and every signable byte. D2 coverage truthfulness checks diff --git a/tests/test_cli.py b/tests/test_cli.py index f777127..73e3bcb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,6 +14,8 @@ import unittest from pathlib import Path +from prototype.conformance import ReferenceConformanceBinding + REPO_ROOT = Path(__file__).resolve().parents[1] SEMANTIC_FIXTURES = REPO_ROOT / "spec" / "semantic" @@ -23,6 +25,13 @@ EXIT_INCONCLUSIVE = 2 +class ExplodingConformanceBinding(ReferenceConformanceBinding): + name = "exploding-test-binding" + + def apply_mutation(self, case, importer, adapter, context) -> None: + raise RuntimeError("deliberate mutation failure") + + class CliCase(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory(prefix="errata-cli-") @@ -64,6 +73,44 @@ def quarantine_and_repair(self) -> subprocess.CompletedProcess[str]: return self.run_cli("repair") +class AdapterConformanceCommand(CliCase): + def run_conformance(self, *extra: str) -> subprocess.CompletedProcess[str]: + return self.run_cli( + "adapter-conformance", + "--corpus", str(REPO_ROOT / "spec" / "adapter-conformance.json"), + "--source-root", str(REPO_ROOT), + *extra, + ) + + def test_reference_binding_emits_canonical_passing_report(self) -> None: + result = self.run_conformance() + self.assertEqual(result.returncode, EXIT_OK, result.stdout + result.stderr) + payload = json.loads(result.stdout) + self.assertTrue(payload["passed"]) + self.assertEqual(len(payload["cases"]), 5) + self.assertEqual(len(payload["validator_controls"]), 3) + self.assertIn("not G2 or G4 evidence", payload["evidence_boundary"]) + self.assertEqual(result.stdout.strip(), json.dumps( + payload, sort_keys=True, separators=(",", ":") + )) + + def test_failed_mutation_control_exits_one(self) -> None: + result = self.run_conformance( + "--binding", "tests.test_cli:ExplodingConformanceBinding" + ) + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertFalse(json.loads(result.stdout)["passed"]) + + def test_invalid_source_evidence_exits_two(self) -> None: + result = self.run_cli( + "adapter-conformance", + "--corpus", str(REPO_ROOT / "spec" / "adapter-conformance.json"), + "--source-root", str(self.cwd), + ) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertIn("invalid conformance evidence", result.stderr) + + class WorkspaceLifecycle(CliCase): def test_commands_refuse_before_init(self) -> None: result = self.run_cli("repair") From 495e4bdfc88b8950d4c037720565ac1912ba966a Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:18:06 +0200 Subject: [PATCH 08/27] release: prepare 0.4.0 adapter conformance hardening --- CHANGELOG.md | 39 ++++++++++++++++++++++++++++- CITATION.cff | 5 ++-- PRODUCTION_READINESS.md | 4 +-- README.md | 4 +-- SECURITY.md | 4 +-- VERSION | 2 +- docs/PUBLICATION_LOG.md | 18 +++++++++++++ readiness/production-readiness.json | 7 ++++-- scripts/check_readiness.py | 6 +++-- tests/test_readiness.py | 38 +++++++++++++++++++++++++--- tests/test_validate_repo.py | 22 ++++++++++++++-- 11 files changed, 128 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13571b9..39a939f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,44 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), Changes to the bounded novelty statement or the source comparison are recorded here even when they narrow or retire part of the claim. That is the intended direction of travel, not an exception. -## [Unreleased](https://github.com/thomaswillner/llm-errata/compare/v0.3.0...HEAD) +## [Unreleased](https://github.com/thomaswillner/llm-errata/compare/v0.4.0...HEAD) + +## [0.4.0](https://github.com/thomaswillner/llm-errata/releases/tag/v0.4.0) - 2026-08-13 + +### Added — adapter conformance and validator hardening + +- Added five provider-neutral adapter cases with complete checkpoint, + aggregate, triad, store, and receipt outcomes. Calls are traced through the + exact target adapter instance rather than matched globally by function name. +- Added bounded proposition-multiplicity preservation. Synthetic conformance + bindings expose stable provider-local proposition identity and active count; + an unobservable count is `unknown`, not a pass. +- Added exact semantic mutation controls. A mutation must complete and produce + its declared counter-result; an exception or different failure cannot earn + credit. +- Added three executable validator attacks covering empty receipts, incomplete + no-op feed acceptance, and constant-`unknown` semantic aggregation. +- Added `errata adapter-conformance`, canonical JSON reporting, and distinct + pass, failed-control, and invalid-evidence exit codes. + +Rastislav Drahos/DanceNitra materially advanced this release by reporting the +adapter-coverage gap, a real duplicate-preservation defect, candidate adapter +behaviors, and the three anti-vacuity attacks. The immutable MIT-licensed source +artifact is +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`; Inspeximus is a G4 +candidate and the source commit discloses Claude Opus 5 co-authorship, so the +input remains interested-party evidence. LLM Errata independently authored its +corpus and validator without copying or vendoring that runner or fixture. + +### Status and ownership + +- Repository files authored here remain copyright Thomas Rainer Willner. + Contributor credit and MIT permission do not transfer copyright. +- Existing licence terms are unchanged: independent commercial and + non-commercial specification implementations are permitted under the + attributed implementation grant; Reference Code remains restricted. +- Verdict remains `NOT_PROD_READY`. G2 through G6 remain blocked by their + existing external evidence requirements. ### Added — production-readiness evidence diff --git a/CITATION.cff b/CITATION.cff index 41ad2e7..b4eb0d7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,8 +5,8 @@ type: software authors: - family-names: "Willner" given-names: "Thomas Rainer" -version: 0.3.0 -date-released: 2026-08-07 +version: 0.4.0 +date-released: 2026-08-13 license: LicenseRef-LLM-Errata-Personal-Use license-url: "https://github.com/thomaswillner/llm-errata/blob/main/LICENSE" abstract: >- @@ -21,4 +21,3 @@ keywords: - provenance - conformance - verifiable repair - diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 30696f2..edcb913 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -4,7 +4,7 @@ | Field | Value | |---|---| -| Version | 0.3.0 | +| Version | 0.4.0 | | Verdict | **NOT_PROD_READY** | | Ledger | `readiness/production-readiness.json` | @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, adapter-level conformance, validator anti-vacuity controls, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | Inspeximus `v2.7.0` is one tagged externally authored adapter candidate with disclosed v2.6.1 reference-code contamination and a claimed clean-room rewrite. It targets historical commit `a477fe4f5c86730031b6285d9505778fb8eec060`; provenance, current-target behavior, a second candidate, and a third-party validator result remain unverified. | Rebind candidates to the current immutable target; obtain dated evidence from two independently authored adapters, including separate provenance review where needed, and a separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/README.md b/README.md index e00561d..22f7c92 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ | Field | Value | |---|---| | Author | Thomas Rainer Willner | -| Version | 0.3.0 | +| Version | 0.4.0 | | Status | Public concept proposal / Request for Comment | | Published | 2026-08-07 | | Research reviewed through | 2026-08-01 | @@ -179,7 +179,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface include conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, and mutation coverage for every signed receipt field. G2 remains `BLOCKED`: the reviewer disclosed a commercial conflict and did not cover the full required conformance scope. G4 also remains `BLOCKED`: one tagged external adapter candidate exists, but its clean-room provenance and behavior are not third-party validated and no separate validator result exists. +Version 0.4.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface include conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, mutation coverage for every signed receipt field, and independently authored adapter-level cases with target-instance tracing, complete outcomes, bounded proposition multiplicity, exact semantic mutations, and executable validator anti-vacuity controls. G2 remains `BLOCKED`: interested-party findings and internal remediation do not replace a complete independent review of the current surface. G4 also remains `BLOCKED`: one externally authored adapter candidate exists, but two independent implementations and a separately produced third-party validator result are not established. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/SECURITY.md b/SECURITY.md index d010920..9f7d86d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,8 +8,8 @@ Until a later policy states otherwise, only the latest versioned release is elig | Version | Supported | |---|---| -| 0.3.x | Yes | -| 0.2.x and earlier | No | +| 0.4.x | Yes | +| 0.3.x and earlier | No | | Unreleased development revisions | No | ## Reporting a vulnerability diff --git a/VERSION b/VERSION index 0d91a54..1d0ba9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.4.0 diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 9e85ba8..753ab73 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -1,5 +1,23 @@ # Publication log +## 2026-08-13 — adapter-conformance feedback incorporated for 0.4.0 + +Rastislav Drahos/DanceNitra published an MIT-licensed candidate fixture at +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. Review reproduced +the reported 5/5 result and found false-pass paths in exception handling, +global name tracing, source binding, citation validation, declarative-only +anti-vacuity cases, and partial expected outcomes. LLM Errata independently +implemented the accepted behaviors with exact target-instance tracing, +complete outcomes, bounded proposition multiplicity, exact mutation results, +and executable validator attacks. The external runner and fixture were not +copied or vendored. + +This is internal remediation from interested-party feedback. It does not +satisfy G2 or G4. Repository ownership and licence terms are unchanged: +Thomas Rainer Willner retains copyright in repository-authored files; +independent specification implementations remain permitted under the +attributed implementation grant; Reference Code remains separately restricted. + This log records publication and outreach evidence. It does not upgrade a readiness gate. Last verified: 2026-08-13. diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index 36d4b2e..467d7e9 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -1,8 +1,8 @@ { "schema_version": 1, - "project_version": "0.3.0", + "project_version": "0.4.0", "verdict": "NOT_PROD_READY", - "last_reviewed": "2026-08-12", + "last_reviewed": "2026-08-13", "gates": [ { "id": "G1", @@ -39,6 +39,9 @@ {"kind": "repository", "ref": "spec/semantic/observations.json"}, {"kind": "repository", "ref": "spec/vectors/protocol-manifest.json"}, {"kind": "repository", "ref": "spec/vectors/receipt-binding-mutations.json"}, + {"kind": "repository", "ref": "spec/adapter-conformance.json"}, + {"kind": "repository", "ref": "prototype/conformance.py"}, + {"kind": "repository", "ref": "tests/test_conformance.py"}, {"kind": "repository", "ref": "prototype/README.md"}, {"kind": "repository", "ref": "spec/README.md"}, {"kind": "repository", "ref": "REVIEW_REQUEST.md"}, diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index ce8f4e5..2cfb77d 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -43,6 +43,7 @@ "tests/test_adapters.py", "tests/test_checkpoints.py", "tests/test_cli.py", + "tests/test_conformance.py", "tests/test_controller.py", "tests/test_ed25519.py", "tests/test_errata_feed.py", @@ -93,8 +94,9 @@ G2_MATRIX_CURRENT_EVIDENCE = ( "Phase 2 implementation includes conflict-disclosed remediation for split-view " "equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract " - "completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, " - "and receipt binding; no qualifying independent review is recorded." + "completeness, plus schemas, semantic probes, adapter-level conformance, validator " + "anti-vacuity controls, key rotation, invalid-target, confidentiality, and receipt binding; " + "no qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( "Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation." diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 011021e..d2c09ff 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -11,6 +11,7 @@ from pathlib import Path from scripts.check_readiness import ( + G2_MATRIX_CURRENT_EVIDENCE, G6_SCOPE, g2_surface_digest, g2_surface_digest_at_commit, @@ -36,6 +37,32 @@ SCRIPT = "check_readiness.py" +class Release040ReadinessBoundary(unittest.TestCase): + def test_release_updates_version_without_upgrading_external_gates(self) -> None: + root = Path(__file__).resolve().parents[1] + payload = json.loads( + (root / "readiness" / "production-readiness.json").read_text() + ) + self.assertEqual(payload["project_version"], "0.4.0") + self.assertEqual(payload["verdict"], "NOT_PROD_READY") + self.assertEqual( + {gate["id"]: gate["status"] for gate in payload["gates"]}, + { + "G1": "PASS", + "G2": "BLOCKED", + "G3": "BLOCKED", + "G4": "BLOCKED", + "G5": "BLOCKED", + "G6": "BLOCKED", + }, + ) + g2 = next(gate for gate in payload["gates"] if gate["id"] == "G2") + refs = {item.get("ref") for item in g2["evidence"]} + self.assertIn("prototype/conformance.py", refs) + self.assertIn("spec/adapter-conformance.json", refs) + self.assertIn("tests/test_conformance.py", refs) + + class ReadinessCheckerPasses(unittest.TestCase): def test_cryptography_qualification_foregrounds_refusal_evidence(self) -> None: qualification = ( @@ -277,12 +304,13 @@ def _complete_review(root: Path, commit: str) -> dict[str, object]: def test_consistently_formatted_canonical_matrix_cells_are_accepted(self) -> None: def mutate(root): + version = (root / "VERSION").read_text(encoding="utf-8").strip() path = root / "PRODUCTION_READINESS.md" lines = path.read_text(encoding="utf-8").splitlines() formatted = [] for line in lines: if line.startswith("| Version |"): - formatted.append("| **Version** | `**0.3.0**` |") + formatted.append(f"| **Version** | `**{version}**` |") elif line.startswith("| Verdict |"): formatted.append("| `Verdict` | __**NOT_PROD_READY**__ |") elif line.startswith("| G"): @@ -364,9 +392,10 @@ def test_project_version_drift_is_rejected(self) -> None: def test_matrix_project_version_contradiction_is_rejected(self) -> None: def mutate(root): + version = (root / "VERSION").read_text(encoding="utf-8").strip() rewrite( root / "PRODUCTION_READINESS.md", - "| Version | 0.3.0 |", + f"| Version | {version} |", "| Version | 9.9.9 |", ) @@ -435,7 +464,7 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded.", + G2_MATRIX_CURRENT_EVIDENCE, "local tests prove readiness.", ) @@ -501,7 +530,8 @@ def mutate(root): def test_missing_matrix_project_version_is_rejected(self) -> None: def mutate(root): - rewrite(root / "PRODUCTION_READINESS.md", "| Version | 0.3.0 |\n", "") + version = (root / "VERSION").read_text(encoding="utf-8").strip() + rewrite(root / "PRODUCTION_READINESS.md", f"| Version | {version} |\n", "") result = check_after(SCRIPT, mutate) self.assert_rejected_without_traceback(result, "matrix project version") diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 8941f69..c018c28 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -17,6 +17,21 @@ SCRIPT = "validate_repo.py" +class Release040Metadata(unittest.TestCase): + def test_version_citation_maturity_security_and_changelog_align(self) -> None: + root = Path(__file__).resolve().parents[1] + self.assertEqual((root / "VERSION").read_text().strip(), "0.4.0") + citation = (root / "CITATION.cff").read_text(encoding="utf-8") + self.assertIn("version: 0.4.0", citation) + self.assertIn("date-released: 2026-08-13", citation) + self.assertIn("Version 0.4.0", (root / "README.md").read_text()) + self.assertIn("| 0.4.x | Yes |", (root / "SECURITY.md").read_text()) + changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") + self.assertIn("## [0.4.0]", changelog) + self.assertIn("Rastislav Drahos", changelog) + self.assertIn("2ba1e299b3483b9038d03387345702427608b90b", changelog) + + class ValidatorPasses(unittest.TestCase): def test_unmodified_repository_passes(self) -> None: with repo_copy() as root: @@ -383,10 +398,13 @@ def mutate(root: Path) -> None: def test_earlier_security_versions_cannot_be_supported(self) -> None: def mutate(root: Path) -> None: + current = (root / "VERSION").read_text(encoding="utf-8").strip() + major, minor, _ = current.split(".") + row = f"| {major}.{int(minor) - 1}.x and earlier |" rewrite( root / "SECURITY.md", - "| 0.2.x and earlier | No |", - "| 0.2.x and earlier | Yes |", + f"{row} No |", + f"{row} Yes |", ) result = check_after(SCRIPT, mutate) From 514ce291dbd760d0bd836eafd1b037dfa4337c38 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:38:57 +0200 Subject: [PATCH 09/27] fix: harden adapter conformance evidence --- SECURITY.md | 10 +- prototype/conformance.py | 386 +++++++++++++++++++++++++++++------- tests/test_cli.py | 7 +- tests/test_conformance.py | 146 ++++++++++++-- tests/test_validate_repo.py | 8 +- 5 files changed, 470 insertions(+), 87 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 9f7d86d..f34fdb9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,12 +4,16 @@ LLM Errata is currently a public concept proposal and may later include schemas, ## Supported versions -Until a later policy states otherwise, only the latest versioned release is eligible for security fixes. Development revisions after that release receive fixes at maintainer discretion and are not represented as supported releases. +Until a later policy states otherwise, only the latest versioned release is +eligible for security fixes. Support moves from 0.3.x to 0.4.x beginning with the immutable `v0.4.0` release. Development revisions after the latest release +receive fixes at maintainer discretion and are not represented as supported +releases. | Version | Supported | |---|---| -| 0.4.x | Yes | -| 0.3.x and earlier | No | +| 0.4.x | Yes, after `v0.4.0` is published | +| 0.3.x | Yes, until `v0.4.0` is published | +| 0.2.x and earlier | No | | Unreleased development revisions | No | ## Reporting a vulnerability diff --git a/prototype/conformance.py b/prototype/conformance.py index e874434..81b4712 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -3,11 +3,13 @@ from __future__ import annotations import hashlib +import inspect import json import re +import signal import subprocess +import threading from dataclasses import dataclass -from enum import Enum from pathlib import Path from typing import Any, Callable @@ -75,6 +77,42 @@ class NormativeTarget: surface_digest: str +@dataclass(frozen=True, order=True) +class PropositionObservation: + """Content-safe multiplicity for one stable provider-local proposition.""" + + proposition_id: str + fixture_label: str + active_count: int + + def __post_init__(self) -> None: + if not self.proposition_id or not self.fixture_label: + raise ConformanceInputError("proposition identity and label must be non-empty") + if isinstance(self.active_count, bool) or self.active_count < 0: + raise ConformanceInputError("proposition active count must be non-negative") + + +def compare_proposition_multiplicity( + before: tuple[PropositionObservation, ...] | None, + after: tuple[PropositionObservation, ...] | None, +) -> str: + """Compare exact stable identities; text similarity is never an identity seam.""" + + if before is None or after is None: + return "unknown" + before_by_id = {item.proposition_id: item for item in before} + after_by_id = {item.proposition_id: item for item in after} + if len(before_by_id) != len(before) or len(after_by_id) != len(after): + return "unknown" + for proposition_id, prior in before_by_id.items(): + current = after_by_id.get(proposition_id) + if current is None or current.fixture_label != prior.fixture_label: + return "unknown" + if current.active_count > prior.active_count: + return "increased" + return "known" + + @dataclass(frozen=True) class AdapterCase: value: dict[str, Any] @@ -154,6 +192,8 @@ class ConformanceReport: normative_commit: str normative_surface_digest: str runtime_commit: str | None + runtime_tree: str + binding_source: dict[str, str] cases: tuple[CaseResult, ...] validator_controls: tuple[ControlResult, ...] provenance: dict[str, str] @@ -175,6 +215,8 @@ def to_dict(self) -> dict[str, Any]: "surface_digest": self.normative_surface_digest, }, "runtime_commit": self.runtime_commit, + "runtime_tree": self.runtime_tree, + "binding_source": dict(sorted(self.binding_source.items())), "cases": [item.to_dict() for item in self.cases], "validator_controls": [ item.to_dict() for item in self.validator_controls @@ -208,6 +250,34 @@ def canonical_json(self) -> str: CONTROL_KEYS = {"id", "mutation", "required_failure"} REQUIRED_TARGET = "ac4468faf73c2cc7949dd29b2a2a151f5bd23116" REQUIRED_DIGEST = "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" +GIT_TIMEOUT_SECONDS = 10.0 +BINDING_TIMEOUT_SECONDS = 10.0 +REQUIRED_PROVENANCE = { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored.", +} +REQUIRED_CONTROLS = ( + { + "id": "empty-receipt-must-fail", + "mutation": "empty-receipt", + "required_failure": "receipt is vacuous", + }, + { + "id": "no-op-feed-verification-must-fail", + "mutation": "no-op-feed-verification", + "required_failure": "accepted feed is incomplete", + }, + { + "id": "constant-unknown-aggregator-must-fail", + "mutation": "constant-unknown-aggregator", + "required_failure": "semantic verdict diversity is missing", + }, +) def _exact(value: object, keys: set[str], label: str) -> dict[str, Any]: @@ -223,9 +293,13 @@ def _nonempty(value: object, label: str) -> str: def _git(root: Path, *args: str) -> bytes: - result = subprocess.run( - ["git", *args], cwd=root, capture_output=True, check=False - ) + try: + result = subprocess.run( + ["git", *args], cwd=root, capture_output=True, check=False, + timeout=GIT_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as error: + raise ConformanceInputError("Git source verification timed out") from error if result.returncode != 0: raise ConformanceInputError("immutable source commit is unavailable") return result.stdout @@ -300,10 +374,8 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: raise ConformanceInputError("normative surface digest does not match source") provenance = _exact(root["provenance"], PROVENANCE_KEYS, "provenance") - if any(not isinstance(value, str) or not value.strip() for value in provenance.values()): - raise ConformanceInputError("provenance fields must be non-empty") - if not re.fullmatch(r"[0-9a-f]{40}", provenance["source_commit"]): - raise ConformanceInputError("provenance source commit must be immutable") + if provenance != REQUIRED_PROVENANCE: + raise ConformanceInputError("provenance does not match the accepted contribution") raw_cases = root["cases"] if not isinstance(raw_cases, list) or len(raw_cases) != 5: @@ -347,6 +419,8 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: if any(not isinstance(item, str) or not item.strip() for item in control.values()): raise ConformanceInputError("validator control fields must be non-empty") controls.append(control) + if tuple(controls) != REQUIRED_CONTROLS: + raise ConformanceInputError("validator controls do not match the executable attacks") return AdapterCorpus( schema_version=1, normative_target=NormativeTarget(target["commit"], target["surface_digest"]), @@ -372,15 +446,28 @@ class ReferenceConformanceAdapter: def __init__(self, *, undeclared: bool = False) -> None: self._records = { - "diet": {"text": "is vegetarian", "inputs": (), "root": "fact:diet"}, + "diet": { + "text": "is vegetarian", "inputs": (), "root": "fact:diet", + "propositions": ("fixture:diet",), + }, "quiet": { - "text": "prefers quiet restaurants", "inputs": (), "root": "fact:quiet" + "text": "prefers quiet restaurants", "inputs": (), "root": "fact:quiet", + "propositions": ("fixture:quiet",), + }, + "budget": { + "text": "moderate budget", "inputs": (), "root": "fact:budget", + "propositions": ("fixture:budget",), + }, + "pet": { + "text": "has a cat", "inputs": (), "root": "fact:pet", + "propositions": ("fixture:pet",), }, - "budget": {"text": "moderate budget", "inputs": (), "root": "fact:budget"}, - "pet": {"text": "has a cat", "inputs": (), "root": "fact:pet"}, "summary": { "text": "is vegetarian; prefers quiet restaurants; moderate budget", "inputs": ("diet", "quiet", "budget"), "root": None, + "propositions": ( + "fixture:diet", "fixture:quiet", "fixture:budget", + ), }, } if undeclared: @@ -388,8 +475,12 @@ def __init__(self, *, undeclared: bool = False) -> None: "text": "synthetic undeclared derivative", "inputs": (), "root": None, + "propositions": (), } self._active = {key: value["text"] for key, value in self._records.items()} + self._active_propositions = { + key: tuple(value["propositions"]) for key, value in self._records.items() + } self._quarantined: set[str] = set() self._retired: set[str] = set() self._rebuilt: set[str] = set() @@ -433,6 +524,7 @@ def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: for target in targets: self._retired.add(target) self._active.pop(target, None) + self._active_propositions.pop(target, None) def rebuild( self, artifact_id: str, *, inputs: tuple[str, ...], replacement: str | None @@ -444,10 +536,19 @@ def rebuild( parts.insert(0, replacement) text = "; ".join(parts) self._active[artifact_id] = text + propositions = tuple( + proposition + for item in inputs + for proposition in self._records[item]["propositions"] + ) + self._active_propositions[artifact_id] = propositions if self._duplicate_inputs: for item in inputs: duplicate = f"duplicate:{item}" self._active[duplicate] = self._records[item]["text"] + self._active_propositions[duplicate] = tuple( + self._records[item]["propositions"] + ) self._rebuilt.add(artifact_id) self._quarantined.discard(artifact_id) return text @@ -485,17 +586,21 @@ def dispositions(self, root: str) -> dict[str, str]: result[item] = "untouched" return result - def proposition_counts(self) -> dict[str, int]: + def proposition_observations(self) -> tuple[PropositionObservation, ...]: counts: dict[str, int] = {} - for text in self._active.values(): - for label, phrase in ( - ("quiet", "prefers quiet restaurants"), - ("budget", "moderate budget"), - ("pet", "has a cat"), - ): - if phrase in text: - counts[label] = counts.get(label, 0) + 1 - return counts + for identities in self._active_propositions.values(): + for proposition_id in identities: + counts[proposition_id] = counts.get(proposition_id, 0) + 1 + labels = { + "fixture:diet": "diet", + "fixture:quiet": "quiet", + "fixture:budget": "budget", + "fixture:pet": "pet", + } + return tuple( + PropositionObservation(proposition_id, labels[proposition_id], count) + for proposition_id, count in sorted(counts.items()) + ) class ReferenceConformanceBinding: @@ -525,7 +630,7 @@ def build(self, case: AdapterCase): return importer, traced, { "adapter": adapter, "owner": owner, - "before_counts": adapter.proposition_counts(), + "before_observations": adapter.proposition_observations(), } def erratum(self, case: AdapterCase, context: dict[str, Any]): @@ -580,11 +685,8 @@ def observe( receipt.to_dict(), sort_keys=True ) triad = dict(receipt.triad) - after = target.proposition_counts() - multiplicity = ( - "increased" - if any(after.get(key, 0) > value for key, value in context["before_counts"].items()) - else "known" + multiplicity = compare_proposition_multiplicity( + context["before_observations"], target.proposition_observations() ) preserved = all( target.recall(term) @@ -638,44 +740,180 @@ def _run_case( return binding.observe(case, importer, adapter, context, checkpoint, receipt), adapter.calls -def run_validator_anti_vacuity_controls() -> tuple[ControlResult, ...]: - """Attack validator acceptance rules, not adapter behavior.""" +def _receipt_errors(value: object) -> tuple[str, ...]: + """Exercise the production receipt schema plus non-vacuity acceptance rule.""" + + from prototype.schema import load as load_schema, validate as validate_schema + + if not isinstance(value, dict) or not value: + return ("receipt is vacuous",) + errors = tuple(validate_schema(value, load_schema("receipt"))) + return errors + + +def _anti_vacuity_receipt( + corpus: AdapterCorpus, receipt_validator: Callable[[object], tuple[str, ...]] +) -> str: + mutation = next( + item for item in corpus.validator_controls if item["mutation"] == "empty-receipt" + ) + errors = tuple(receipt_validator({})) + return mutation["required_failure"] if mutation["required_failure"] in errors else "" - # These inputs are intentionally minimal demonstrations of each historical - # false pass. The acceptance predicate names the semantic evidence that is - # absent instead of treating any exception or mismatch as success. - receipt = {} - empty_failure = "receipt is vacuous" if not receipt else "" - offered = ("event-1", "event-2") - accepted = offered[:1] # no-op/partial verifier failed to return every event - feed_failure = "accepted feed is incomplete" if accepted != offered else "" +def _anti_vacuity_feed( + corpus: AdapterCorpus, feed_verifier: Callable[..., list[object]] +) -> str: + from prototype.errata import Erratum, Operation, RootRegistry + from prototype.signing import Ed25519Signer - verdicts = ("unknown",) * 8 - semantic_failure = ( - "semantic verdict diversity is missing" - if len(set(verdicts)) < 3 else "" + mutation = next( + item for item in corpus.validator_controls + if item["mutation"] == "no-op-feed-verification" ) - return ( - ControlResult("empty-receipt-must-fail", bool(empty_failure), empty_failure), - ControlResult( - "no-op-feed-verification-must-fail", bool(feed_failure), feed_failure - ), - ControlResult( - "constant-unknown-aggregator-must-fail", - bool(semantic_failure), - semantic_failure, - ), + owner = Ed25519Signer(b"conformance-validator-feed") + event = owner.sign_erratum( + Erratum( + erratum_id="anti-vacuity-gap", + sequence=2, + target_root="fact:diet", + operation=Operation.SUPERSEDE, + valid_from="2026-08-01T00:00:00Z", + replacement="eats meat again", + postconditions={ + "negative": "vegetarian", + "positive": "eats meat again", + "preserve": "quiet restaurants", + }, + ) ) + try: + feed_verifier([event], owner=owner.public, roots=RootRegistry({"fact:diet"})) + except Exception as error: + message = str(error) + return mutation["required_failure"] if "gap" in message else "" + return "" -def _runtime_commit(root: Path) -> str | None: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, - check=False, +def _semantic_fixture(source_root: Path, case_name: str): + from prototype.semantic import ( + RecordedSemanticVerifier, SemanticObservation, SemanticProbe, VerifierConfig, ) - value = result.stdout.strip() - return value if result.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}", value) else None + + semantic_root = source_root / "spec" / "semantic" + probes_payload = json.loads((semantic_root / "probes.json").read_text())["cases"] + observations_payload = json.loads( + (semantic_root / "observations.json").read_text() + )["cases"] + config = VerifierConfig.from_dict( + json.loads((semantic_root / "verifier-config.json").read_text()) + ) + probes = tuple(SemanticProbe.from_dict(item) for item in probes_payload[case_name]) + observations = tuple( + SemanticObservation.from_dict(item) for item in observations_payload[case_name] + ) + return probes, config, RecordedSemanticVerifier(observations) + + +def _anti_vacuity_semantic( + corpus: AdapterCorpus, source_root: Path, semantic_runner_factory: Callable[[], object] +) -> str: + from prototype.semantic import SemanticCoverage + + mutation = next( + item for item in corpus.validator_controls + if item["mutation"] == "constant-unknown-aggregator" + ) + runner = semantic_runner_factory() + coverages = { + runner.run(*_semantic_fixture(source_root, case_name)).coverage + for case_name in ("verified-correction", "failed-supersession", "unknown-erasure") + } + required = { + SemanticCoverage.VERIFIED, SemanticCoverage.FAILED, SemanticCoverage.UNKNOWN, + } + return mutation["required_failure"] if coverages == required else "" + + +def run_validator_anti_vacuity_controls( + corpus: AdapterCorpus, + source_root: Path, + *, + receipt_validator: Callable[[object], tuple[str, ...]] = _receipt_errors, + feed_verifier: Callable[..., list[object]] | None = None, + semantic_runner_factory: Callable[[], object] | None = None, +) -> tuple[ControlResult, ...]: + """Install declared flattering mutations against actual acceptance seams.""" + + from prototype.errata import verify_feed + from prototype.semantic import SemanticProbeRunner + + feed = feed_verifier or verify_feed + semantic = semantic_runner_factory or SemanticProbeRunner + failures = ( + _anti_vacuity_receipt(corpus, receipt_validator), + _anti_vacuity_feed(corpus, feed), + _anti_vacuity_semantic(corpus, source_root, semantic), + ) + return tuple( + ControlResult(control["id"], failure == control["required_failure"], failure) + for control, failure in zip(corpus.validator_controls, failures) + ) + + +def _runtime_identity(root: Path) -> tuple[str, str]: + status = _git(root, "status", "--porcelain=v1", "--untracked-files=all") + if status.strip(): + raise ConformanceInputError("runtime source tree is dirty") + commit = _git(root, "rev-parse", "HEAD").decode().strip() + tree = _git(root, "rev-parse", "HEAD^{tree}").decode().strip() + if not re.fullmatch(r"[0-9a-f]{40}", commit) or not re.fullmatch( + r"[0-9a-f]{40}", tree + ): + raise ConformanceInputError("runtime source identity is invalid") + return commit, tree + + +def _binding_source( + source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding] +) -> dict[str, str]: + try: + file_path = Path(inspect.getsourcefile(binding_factory) or "").resolve() + relative = file_path.relative_to(source_root.resolve()).as_posix() + payload = file_path.read_bytes() + except (OSError, ValueError) as error: + raise ConformanceInputError( + "binding source is not inside the declared source root" + ) from error + if relative.startswith(".git/"): + raise ConformanceInputError("binding source is not executable repository source") + tracked = set( + _git(source_root, "ls-files", "--cached").decode("utf-8").splitlines() + ) + if relative not in tracked: + raise ConformanceInputError("binding source is not tracked by the runtime tree") + return {"path": relative, "sha256": hashlib.sha256(payload).hexdigest()} + + +class _BindingTimeout(Exception): + pass + + +def _run_with_timeout(action: Callable[[], Any]) -> Any: + if threading.current_thread() is not threading.main_thread(): + return action() + + def expire(signum: int, frame: object) -> None: + raise _BindingTimeout + + prior = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, BINDING_TIMEOUT_SECONDS) + try: + return action() + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, prior) def validate_adapter_conformance( @@ -683,28 +921,40 @@ def validate_adapter_conformance( source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding], ) -> ConformanceReport: + runtime_commit, runtime_tree = _runtime_identity(source_root) corpus = load_corpus(corpus_path, source_root) + binding_source = _binding_source(source_root, binding_factory) binding = binding_factory() results = [] for case in corpus.cases: try: - observed, calls = _run_case(binding, case, mutate=False) + observed, calls = _run_with_timeout( + lambda case=case: _run_case(binding, case, mutate=False) + ) failures = compare_complete_outcome(case.value["expected"], observed) missing = tuple(sorted(set(case.value["required_calls"]) - set(calls))) + except _BindingTimeout as error: + raise ConformanceInputError("binding execution timed out") from error except Exception as error: - observed, calls, missing = {}, (), tuple(case.value["required_calls"]) - failures = (f"honest run raised unexpected {type(error).__name__}: {error}",) + raise ConformanceInputError( + f"binding execution failed for {case.case_id}: {type(error).__name__}" + ) from error mutation_observed = None mutation_failures: tuple[str, ...] try: - mutation_observed, _ = _run_case(binding, case, mutate=True) + mutation_observed, _ = _run_with_timeout( + lambda case=case: _run_case(binding, case, mutate=True) + ) mutation_failures = compare_complete_outcome( case.value["mutation"]["exact_counter_result"], mutation_observed ) + except _BindingTimeout as error: + raise ConformanceInputError("binding execution timed out") from error except Exception as error: - mutation_failures = ( - f"mutation raised unexpected {type(error).__name__}: {error}", - ) + raise ConformanceInputError( + f"binding execution failed for {case.case_id} mutation: " + f"{type(error).__name__}" + ) from error results.append( CaseResult( case_id=case.case_id, @@ -724,9 +974,11 @@ def validate_adapter_conformance( binding=binding.name, normative_commit=corpus.normative_target.commit, normative_surface_digest=corpus.normative_target.surface_digest, - runtime_commit=_runtime_commit(source_root), + runtime_commit=runtime_commit, + runtime_tree=runtime_tree, + binding_source=binding_source, cases=tuple(results), - validator_controls=run_validator_anti_vacuity_controls(), + validator_controls=run_validator_anti_vacuity_controls(corpus, source_root), provenance=corpus.provenance, evidence_boundary=corpus.evidence_boundary, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 73e3bcb..aec115c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -94,12 +94,13 @@ def test_reference_binding_emits_canonical_passing_report(self) -> None: payload, sort_keys=True, separators=(",", ":") )) - def test_failed_mutation_control_exits_one(self) -> None: + def test_binding_execution_fault_exits_two(self) -> None: result = self.run_conformance( "--binding", "tests.test_cli:ExplodingConformanceBinding" ) - self.assertEqual(result.returncode, EXIT_REFUSED) - self.assertFalse(json.loads(result.stdout)["passed"]) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertEqual(result.stdout, "") + self.assertIn("invalid conformance evidence", result.stderr) def test_invalid_source_evidence_exits_two(self) -> None: result = self.run_cli( diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 3afb82c..6868ac4 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -7,15 +7,21 @@ from __future__ import annotations import json +import subprocess import tempfile +import time import unittest from pathlib import Path +from unittest.mock import patch from prototype.conformance import ( ConformanceInputError, + PropositionObservation, + ReferenceConformanceAdapter, ReferenceConformanceBinding, TracingAdapter, compare_complete_outcome, + compare_proposition_multiplicity, load_corpus, run_validator_anti_vacuity_controls, validate_adapter_conformance, @@ -83,6 +89,27 @@ def test_empty_provenance_is_refused(self) -> None: with self.assertRaisesRegex(ConformanceInputError, "provenance"): load_corpus(path, ROOT) + def test_every_provenance_field_is_bound_to_the_accepted_contribution(self) -> None: + expected = { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored.", + } + self.assertEqual(load_corpus(CORPUS, ROOT).provenance, expected) + for field in expected: + with self.subTest(field=field): + path = self.changed_corpus( + lambda value, field=field: value["provenance"].__setitem__( + field, "plausible but wrong" + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "provenance"): + load_corpus(path, ROOT) + class TargetTracing(unittest.TestCase): def test_only_calls_through_wrapped_target_are_recorded(self) -> None: @@ -163,6 +190,38 @@ def test_missing_or_extra_fields_are_rejected(self) -> None: self.assertIn("store.unexpected: unexpected", failures) +class PropositionMultiplicity(unittest.TestCase): + def test_reference_observations_use_stable_provider_local_ids_and_counts(self) -> None: + observations = ReferenceConformanceAdapter().proposition_observations() + self.assertEqual( + observations, + ( + PropositionObservation("fixture:budget", "budget", 2), + PropositionObservation("fixture:diet", "diet", 2), + PropositionObservation("fixture:pet", "pet", 1), + PropositionObservation("fixture:quiet", "quiet", 2), + ), + ) + + def test_text_equivalence_or_substrings_cannot_create_proposition_identity(self) -> None: + adapter = ReferenceConformanceAdapter() + before = adapter.proposition_observations() + adapter._active["text-only-alias"] = ( + "prefix prefers quiet restaurants suffix; moderate budget-ish" + ) + after = adapter.proposition_observations() + self.assertEqual(compare_proposition_multiplicity(before, after), "known") + + def test_unavailable_identity_or_count_is_unknown(self) -> None: + before = (PropositionObservation("fixture:quiet", "quiet", 2),) + self.assertEqual(compare_proposition_multiplicity(before, None), "unknown") + + def test_exact_identity_count_increase_is_detected(self) -> None: + before = (PropositionObservation("fixture:quiet", "quiet", 2),) + after = (PropositionObservation("fixture:quiet", "quiet", 3),) + self.assertEqual(compare_proposition_multiplicity(before, after), "increased") + + class AdapterCases(unittest.TestCase): def test_reference_binding_passes_five_cases_and_exact_mutations(self) -> None: report = validate_adapter_conformance( @@ -175,29 +234,63 @@ def test_reference_binding_passes_five_cases_and_exact_mutations(self) -> None: self.assertTrue(result.positive_control_passed, result.missing_calls) self.assertTrue(result.mutation_control_passed, result.mutation_failures) - def test_mutation_exception_is_failed_control_not_success(self) -> None: + def test_binding_execution_fault_is_invalid_evidence(self) -> None: class ExplodingBinding(ReferenceConformanceBinding): name = "exploding" def apply_mutation(self, case, importer, adapter, context) -> None: raise RuntimeError("mutation setup broke") - report = validate_adapter_conformance(CORPUS, ROOT, ExplodingBinding) - self.assertFalse(report.passed) - self.assertTrue( - all(not result.mutation_control_passed for result in report.cases) - ) - self.assertTrue( - all( - any("unexpected RuntimeError" in item for item in result.mutation_failures) - for result in report.cases - ) - ) + with self.assertRaisesRegex(ConformanceInputError, "binding execution"): + validate_adapter_conformance(CORPUS, ROOT, ExplodingBinding) + + def test_binding_execution_has_a_hard_timeout(self) -> None: + class SlowBinding(ReferenceConformanceBinding): + def build(self, case): + time.sleep(0.1) + return super().build(case) + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + validate_adapter_conformance(CORPUS, ROOT, SlowBinding) + + +class RuntimeSourceIdentity(unittest.TestCase): + def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None: + report = validate_adapter_conformance(CORPUS, ROOT, ReferenceConformanceBinding) + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, + capture_output=True, text=True, + ).stdout.strip() + tree = subprocess.run( + ["git", "rev-parse", "HEAD^{tree}"], cwd=ROOT, check=True, + capture_output=True, text=True, + ).stdout.strip() + self.assertEqual(report.runtime_commit, commit) + self.assertEqual(report.runtime_tree, tree) + self.assertEqual(report.binding_source["path"], "prototype/conformance.py") + self.assertRegex(report.binding_source["sha256"], r"^[0-9a-f]{64}$") + + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: + marker = ROOT / "conformance-dirty-sentinel.tmp" + marker.write_text("untracked runtime input", encoding="utf-8") + self.addCleanup(marker.unlink, missing_ok=True) + with self.assertRaisesRegex(ConformanceInputError, "dirty"): + validate_adapter_conformance(CORPUS, ROOT, ReferenceConformanceBinding) + + def test_git_timeout_is_invalid_source_evidence(self) -> None: + with patch( + "prototype.conformance.subprocess.run", + side_effect=subprocess.TimeoutExpired(["git"], 5), + ): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + load_corpus(CORPUS, ROOT) class AntiVacuity(unittest.TestCase): def test_all_declared_validator_attacks_are_rejected(self) -> None: - controls = run_validator_anti_vacuity_controls() + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls(corpus, ROOT) self.assertEqual( [item.control_id for item in controls], [ @@ -216,5 +309,32 @@ def test_all_declared_validator_attacks_are_rejected(self) -> None: ], ) + def test_permissive_receipt_validator_makes_empty_receipt_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, receipt_validator=lambda value: (), + ) + self.assertFalse(controls[0].passed) + + def test_no_op_feed_verifier_makes_acceptance_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, feed_verifier=lambda errata, **kwargs: list(errata), + ) + self.assertFalse(controls[1].passed) + + def test_constant_unknown_semantic_runner_makes_diversity_control_fail(self) -> None: + from prototype.semantic import SemanticCoverage + + class ConstantUnknownRunner: + def run(self, probes, config, verifier): + return type("Report", (), {"coverage": SemanticCoverage.UNKNOWN})() + + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, semantic_runner_factory=ConstantUnknownRunner, + ) + self.assertFalse(controls[2].passed) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index c018c28..e017682 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -25,11 +25,17 @@ def test_version_citation_maturity_security_and_changelog_align(self) -> None: self.assertIn("version: 0.4.0", citation) self.assertIn("date-released: 2026-08-13", citation) self.assertIn("Version 0.4.0", (root / "README.md").read_text()) - self.assertIn("| 0.4.x | Yes |", (root / "SECURITY.md").read_text()) + self.assertIn( + "| 0.4.x | Yes, after `v0.4.0` is published |", + (root / "SECURITY.md").read_text(), + ) changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") self.assertIn("## [0.4.0]", changelog) self.assertIn("Rastislav Drahos", changelog) self.assertIn("2ba1e299b3483b9038d03387345702427608b90b", changelog) + security = (root / "SECURITY.md").read_text(encoding="utf-8") + self.assertIn("beginning with the immutable `v0.4.0` release", security) + self.assertIn("until `v0.4.0` is published", security) class ValidatorPasses(unittest.TestCase): From 143846ec0421cea33e0b431b621fc3fe9e80784d Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:39:48 +0200 Subject: [PATCH 10/27] fix: scope multiplicity to preserved identities --- prototype/conformance.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 81b4712..3afc63d 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -607,6 +607,19 @@ class ReferenceConformanceBinding: """Reference binding; external implementations supply an equivalent class.""" name = "llm-errata-reference" + preserved_proposition_ids = frozenset( + {"fixture:quiet", "fixture:budget", "fixture:pet"} + ) + + @classmethod + def preserved_observations( + cls, adapter: ReferenceConformanceAdapter + ) -> tuple[PropositionObservation, ...]: + return tuple( + item + for item in adapter.proposition_observations() + if item.proposition_id in cls.preserved_proposition_ids + ) def build(self, case: AdapterCase): from prototype.controller import Importer @@ -630,7 +643,7 @@ def build(self, case: AdapterCase): return importer, traced, { "adapter": adapter, "owner": owner, - "before_observations": adapter.proposition_observations(), + "before_observations": self.preserved_observations(adapter), } def erratum(self, case: AdapterCase, context: dict[str, Any]): @@ -686,7 +699,7 @@ def observe( ) triad = dict(receipt.triad) multiplicity = compare_proposition_multiplicity( - context["before_observations"], target.proposition_observations() + context["before_observations"], self.preserved_observations(target) ) preserved = all( target.recall(term) From 59d0f91655c5c15a8040de9a5a1db65e2a304860 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:40:28 +0200 Subject: [PATCH 11/27] fix: retain zero-count proposition identities --- prototype/conformance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 3afc63d..1375318 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -587,16 +587,16 @@ def dispositions(self, root: str) -> dict[str, str]: return result def proposition_observations(self) -> tuple[PropositionObservation, ...]: - counts: dict[str, int] = {} - for identities in self._active_propositions.values(): - for proposition_id in identities: - counts[proposition_id] = counts.get(proposition_id, 0) + 1 labels = { "fixture:diet": "diet", "fixture:quiet": "quiet", "fixture:budget": "budget", "fixture:pet": "pet", } + counts = {proposition_id: 0 for proposition_id in labels} + for identities in self._active_propositions.values(): + for proposition_id in identities: + counts[proposition_id] = counts.get(proposition_id, 0) + 1 return tuple( PropositionObservation(proposition_id, labels[proposition_id], count) for proposition_id, count in sorted(counts.items()) From 0ddef193d68e89390b48dbffdcd0e9550967e3e4 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:41:10 +0200 Subject: [PATCH 12/27] test: require bounded binding construction --- tests/test_conformance.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 6868ac4..9f82ae4 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -254,6 +254,15 @@ def build(self, case): with self.assertRaisesRegex(ConformanceInputError, "timed out"): validate_adapter_conformance(CORPUS, ROOT, SlowBinding) + def test_binding_constructor_has_a_hard_timeout(self) -> None: + class SlowConstructorBinding(ReferenceConformanceBinding): + def __init__(self): + time.sleep(0.1) + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + validate_adapter_conformance(CORPUS, ROOT, SlowConstructorBinding) + class RuntimeSourceIdentity(unittest.TestCase): def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None: From 046dcaa876175c56ffbf824f361091fb2529b7a0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:41:26 +0200 Subject: [PATCH 13/27] fix: bound binding construction time --- prototype/conformance.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 1375318..2862dae 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -914,7 +914,9 @@ class _BindingTimeout(Exception): def _run_with_timeout(action: Callable[[], Any]) -> Any: if threading.current_thread() is not threading.main_thread(): - return action() + raise ConformanceInputError( + "binding execution requires the main thread for timeout enforcement" + ) def expire(signum: int, frame: object) -> None: raise _BindingTimeout @@ -937,7 +939,16 @@ def validate_adapter_conformance( runtime_commit, runtime_tree = _runtime_identity(source_root) corpus = load_corpus(corpus_path, source_root) binding_source = _binding_source(source_root, binding_factory) - binding = binding_factory() + try: + binding = _run_with_timeout(binding_factory) + except _BindingTimeout as error: + raise ConformanceInputError("binding construction timed out") from error + except ConformanceInputError: + raise + except Exception as error: + raise ConformanceInputError( + f"binding construction failed: {type(error).__name__}" + ) from error results = [] for case in corpus.cases: try: From b76814b22a3607a2775ff807c8335d5aa74a1fa3 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:44:55 +0200 Subject: [PATCH 14/27] fix: validate pre-release security transition --- scripts/validate_repo.py | 28 +++++++++++++++++----------- tests/test_validate_repo.py | 7 ++++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 9fc95ec..b928f93 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -593,23 +593,29 @@ def check_document_version_alignment( cells = [cell.strip() for cell in row.split("|")] if len(cells) == 4 and cells[0] == cells[-1] == "": supported_rows.append((cells[1], cells[2])) - supported_yes = [ - version - for version, status in supported_rows - if status == "Yes" - ] previous_minor = int(minor) - 1 - required_unsupported_rows = { + released_rows = { + (f"{major}.{minor}.x", "Yes"), (f"{major}.{previous_minor}.x and earlier", "No"), ("Unreleased development revisions", "No"), } + prepared_rows = { + (f"{major}.{minor}.x", f"Yes, after `v{repository_version}` is published"), + ( + f"{major}.{previous_minor}.x", + f"Yes, until `v{repository_version}` is published", + ), + (f"{major}.{previous_minor - 1}.x and earlier", "No"), + ("Unreleased development revisions", "No"), + } + supported_set = frozenset(supported_rows) reporter.check( "SECURITY supported version", - supported_yes == [f"{major}.{minor}.x"] - and required_unsupported_rows.issubset(set(supported_rows)), - f"SECURITY.md supports {major}.{minor}.x", - "Keep exactly one Yes row for VERSION major.minor.x and No rows for " - "the prior-version family and unreleased revisions.", + supported_set in {frozenset(released_rows), frozenset(prepared_rows)} + and len(supported_rows) == len(supported_set), + f"SECURITY.md is either release-prepared for or supports {major}.{minor}.x", + "Use exactly the conditional pre-release transition rows or the final " + "released VERSION row plus prior/unreleased No rows.", ) diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index e017682..4b0d286 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -385,7 +385,8 @@ def test_security_supported_version_drift_is_rejected(self) -> None: def mutate(root: Path) -> None: current = (root / "VERSION").read_text(encoding="utf-8").strip() major, minor, _ = current.split(".") - rewrite(root / "SECURITY.md", f"| {major}.{minor}.x | Yes |", "| 0.0.x | Yes |") + expected = f"| {major}.{minor}.x | Yes, after `v{current}` is published |" + rewrite(root / "SECURITY.md", expected, "| 0.0.x | Yes |") result = check_after(SCRIPT, mutate) self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) @@ -395,7 +396,7 @@ def test_extra_security_supported_version_is_rejected(self) -> None: def mutate(root: Path) -> None: current = (root / "VERSION").read_text(encoding="utf-8").strip() major, minor, _ = current.split(".") - supported = f"| {major}.{minor}.x | Yes |" + supported = f"| {major}.{minor}.x | Yes, after `v{current}` is published |" rewrite(root / "SECURITY.md", supported, supported + "\n| 0.2.x | Yes |") result = check_after(SCRIPT, mutate) @@ -406,7 +407,7 @@ def test_earlier_security_versions_cannot_be_supported(self) -> None: def mutate(root: Path) -> None: current = (root / "VERSION").read_text(encoding="utf-8").strip() major, minor, _ = current.split(".") - row = f"| {major}.{int(minor) - 1}.x and earlier |" + row = f"| {major}.{int(minor) - 2}.x and earlier |" rewrite( root / "SECURITY.md", f"{row} No |", From d79bfd8db3e293bfe39783e975e789d7d221f8ff Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:55:59 +0200 Subject: [PATCH 15/27] test: isolate dirty-tree conformance fixture --- tests/test_conformance.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 9f82ae4..c4fa213 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -281,11 +281,26 @@ def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None self.assertRegex(report.binding_source["sha256"], r"^[0-9a-f]{64}$") def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: - marker = ROOT / "conformance-dirty-sentinel.tmp" - marker.write_text("untracked runtime input", encoding="utf-8") - self.addCleanup(marker.unlink, missing_ok=True) - with self.assertRaisesRegex(ConformanceInputError, "dirty"): - validate_adapter_conformance(CORPUS, ROOT, ReferenceConformanceBinding) + with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: + source_root = Path(directory) + subprocess.run(["git", "init", "-q"], cwd=source_root, check=True) + marker = source_root / "tracked.txt" + marker.write_text("clean\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=source_root, check=True) + subprocess.run( + [ + "git", "-c", "user.name=LLM Errata Tests", + "-c", "user.email=tests@example.invalid", + "commit", "-q", "-m", "fixture", + ], + cwd=source_root, + check=True, + ) + marker.write_text("dirty\n", encoding="utf-8") + with self.assertRaisesRegex(ConformanceInputError, "dirty"): + validate_adapter_conformance( + CORPUS, source_root, ReferenceConformanceBinding + ) def test_git_timeout_is_invalid_source_evidence(self) -> None: with patch( From 0eff23efaa65efd7e3568e58953c90c6fe4f3241 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:57:36 +0200 Subject: [PATCH 16/27] test: bind receipt and metadata acceptance seams --- tests/test_cli.py | 4 ++++ tests/test_conformance.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index aec115c..d105db2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,6 +15,7 @@ from pathlib import Path from prototype.conformance import ReferenceConformanceBinding +from prototype.receipts import receipt_acceptance_errors REPO_ROOT = Path(__file__).resolve().parents[1] @@ -230,6 +231,9 @@ def test_plan_changes_nothing(self) -> None: class ReceiptsAreVerifiable(CliCase): + def test_empty_receipt_is_rejected_by_shared_acceptance_seam(self) -> None: + self.assertIn("receipt is vacuous", receipt_acceptance_errors({})) + def test_a_genuine_receipt_verifies(self) -> None: self.seed() self.publish_supersession() diff --git a/tests/test_conformance.py b/tests/test_conformance.py index c4fa213..620b286 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -263,6 +263,17 @@ def __init__(self): with self.assertRaisesRegex(ConformanceInputError, "timed out"): validate_adapter_conformance(CORPUS, ROOT, SlowConstructorBinding) + def test_binding_metadata_has_a_hard_timeout(self) -> None: + class SlowMetadataBinding(ReferenceConformanceBinding): + @property + def name(self): + time.sleep(0.1) + return "slow-metadata" + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + validate_adapter_conformance(CORPUS, ROOT, SlowMetadataBinding) + class RuntimeSourceIdentity(unittest.TestCase): def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None: @@ -340,6 +351,12 @@ def test_permissive_receipt_validator_makes_empty_receipt_control_fail(self) -> ) self.assertFalse(controls[0].passed) + def test_permissive_production_receipt_schema_makes_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + with patch("prototype.receipts.validate_schema", return_value=[]): + controls = run_validator_anti_vacuity_controls(corpus, ROOT) + self.assertFalse(controls[0].passed) + def test_no_op_feed_verifier_makes_acceptance_control_fail(self) -> None: corpus = load_corpus(CORPUS, ROOT) controls = run_validator_anti_vacuity_controls( From 73a2b9dce47fc8e7b2053399c6f72b3f4b68e16e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:59:35 +0200 Subject: [PATCH 17/27] fix: share receipt acceptance and bound metadata --- prototype/cli.py | 9 ++++++--- prototype/conformance.py | 31 +++++++++++++++++-------------- prototype/receipts.py | 9 +++++++++ tests/test_cli.py | 9 +++++++++ tests/test_conformance.py | 6 ++++-- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index f67fc2b..4a7e674 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -37,7 +37,7 @@ from prototype.controller import Importer, Phase from prototype.errata import Erratum, FeedError, Operation, RootRegistry, read_feed from prototype.lineage import LineageLedger -from prototype.receipts import Receipt +from prototype.receipts import Receipt, receipt_acceptance_errors from prototype.schema import load as load_schema, validate as validate_schema from prototype.semantic import ( RecordedSemanticVerifier, @@ -358,10 +358,13 @@ def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: return EXIT_REFUSED key = ws.importer_verification_key() - schema = load_schema("receipt") bad = 0 for name, payload in receipts: - errors = validate_schema(payload, schema) + errors = receipt_acceptance_errors(payload) + if errors: + bad += 1 + print(f" {name}: signature=not-checked schema={errors[0]} -> BAD") + continue signature = payload.get("signature") rebuilt = Receipt( importer=payload["importer"], diff --git a/prototype/conformance.py b/prototype/conformance.py index 2862dae..9d3f7c7 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -753,17 +753,6 @@ def _run_case( return binding.observe(case, importer, adapter, context, checkpoint, receipt), adapter.calls -def _receipt_errors(value: object) -> tuple[str, ...]: - """Exercise the production receipt schema plus non-vacuity acceptance rule.""" - - from prototype.schema import load as load_schema, validate as validate_schema - - if not isinstance(value, dict) or not value: - return ("receipt is vacuous",) - errors = tuple(validate_schema(value, load_schema("receipt"))) - return errors - - def _anti_vacuity_receipt( corpus: AdapterCorpus, receipt_validator: Callable[[object], tuple[str, ...]] ) -> str: @@ -852,19 +841,21 @@ def run_validator_anti_vacuity_controls( corpus: AdapterCorpus, source_root: Path, *, - receipt_validator: Callable[[object], tuple[str, ...]] = _receipt_errors, + receipt_validator: Callable[[object], tuple[str, ...]] | None = None, feed_verifier: Callable[..., list[object]] | None = None, semantic_runner_factory: Callable[[], object] | None = None, ) -> tuple[ControlResult, ...]: """Install declared flattering mutations against actual acceptance seams.""" from prototype.errata import verify_feed + from prototype.receipts import receipt_acceptance_errors from prototype.semantic import SemanticProbeRunner + receipt = receipt_validator or receipt_acceptance_errors feed = feed_verifier or verify_feed semantic = semantic_runner_factory or SemanticProbeRunner failures = ( - _anti_vacuity_receipt(corpus, receipt_validator), + _anti_vacuity_receipt(corpus, receipt), _anti_vacuity_feed(corpus, feed), _anti_vacuity_semantic(corpus, source_root, semantic), ) @@ -949,6 +940,18 @@ def validate_adapter_conformance( raise ConformanceInputError( f"binding construction failed: {type(error).__name__}" ) from error + try: + binding_name = _run_with_timeout(lambda: binding.name) + except _BindingTimeout as error: + raise ConformanceInputError("binding metadata timed out") from error + except ConformanceInputError: + raise + except Exception as error: + raise ConformanceInputError( + f"binding metadata failed: {type(error).__name__}" + ) from error + if not isinstance(binding_name, str) or not binding_name.strip(): + raise ConformanceInputError("binding metadata name must be non-empty") results = [] for case in corpus.cases: try: @@ -995,7 +998,7 @@ def validate_adapter_conformance( ) ) return ConformanceReport( - binding=binding.name, + binding=binding_name, normative_commit=corpus.normative_target.commit, normative_surface_digest=corpus.normative_target.surface_digest, runtime_commit=runtime_commit, diff --git a/prototype/receipts.py b/prototype/receipts.py index 2f0830d..92644f8 100644 --- a/prototype/receipts.py +++ b/prototype/receipts.py @@ -20,9 +20,18 @@ from typing import Any, Mapping, Sequence from prototype.adapters import Coverage +from prototype.schema import load as load_schema, validate as validate_schema from prototype.signing import VerificationKey +def receipt_acceptance_errors(value: object) -> tuple[str, ...]: + """Return production receipt-acceptance errors, including non-vacuity.""" + + if not isinstance(value, dict) or not value: + return ("receipt is vacuous",) + return tuple(validate_schema(value, load_schema("receipt"))) + + def aggregate_coverage( stores: Mapping[str, Coverage], triad: Mapping[str, str] ) -> Coverage: diff --git a/tests/test_cli.py b/tests/test_cli.py index d105db2..07eb465 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -240,6 +240,15 @@ def test_a_genuine_receipt_verifies(self) -> None: self.quarantine_and_repair() self.assertEqual(self.run_cli("verify").returncode, EXIT_OK) + def test_empty_receipt_is_refused_without_traceback(self) -> None: + self.seed() + path = self.cwd / ".errata" / "receipts" / "empty.json" + path.write_text("{}", encoding="utf-8") + result = self.run_cli("verify") + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertIn("receipt is vacuous", result.stdout) + self.assertNotIn("Traceback", result.stderr) + def test_a_tampered_aggregate_is_caught(self) -> None: self.seed() self.publish_supersession() diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 620b286..dfea682 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -271,7 +271,9 @@ def name(self): return "slow-metadata" with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): - with self.assertRaisesRegex(ConformanceInputError, "timed out"): + with self.assertRaisesRegex( + ConformanceInputError, "binding metadata timed out" + ): validate_adapter_conformance(CORPUS, ROOT, SlowMetadataBinding) @@ -353,7 +355,7 @@ def test_permissive_receipt_validator_makes_empty_receipt_control_fail(self) -> def test_permissive_production_receipt_schema_makes_control_fail(self) -> None: corpus = load_corpus(CORPUS, ROOT) - with patch("prototype.receipts.validate_schema", return_value=[]): + with patch("prototype.receipts.receipt_acceptance_errors", return_value=()): controls = run_validator_anti_vacuity_controls(corpus, ROOT) self.assertFalse(controls[0].passed) From 120cda881cf2f76169d28feb7228429146103d09 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:06:47 +0200 Subject: [PATCH 18/27] test: reject malformed receipt JSON safely --- tests/test_cli.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 07eb465..fe18e95 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -249,6 +249,15 @@ def test_empty_receipt_is_refused_without_traceback(self) -> None: self.assertIn("receipt is vacuous", result.stdout) self.assertNotIn("Traceback", result.stderr) + def test_malformed_receipt_json_is_refused_without_traceback(self) -> None: + self.seed() + path = self.cwd / ".errata" / "receipts" / "malformed.json" + path.write_text("{not json", encoding="utf-8") + result = self.run_cli("verify") + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertIn("receipt JSON is unreadable", result.stdout) + self.assertNotIn("Traceback", result.stderr) + def test_a_tampered_aggregate_is_caught(self) -> None: self.seed() self.publish_supersession() From 3d5dc890e47977f45500f689fc9667b052f14936 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:07:33 +0200 Subject: [PATCH 19/27] fix: refuse unreadable receipt evidence --- prototype/cli.py | 8 ++++++-- prototype/workspace.py | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index 4a7e674..b22293e 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -49,7 +49,7 @@ ) from prototype.signing import Ed25519Signer from prototype.sqlite_store import SqliteAdapter -from prototype.workspace import Workspace +from prototype.workspace import ReceiptReadError, Workspace EXIT_OK = 0 @@ -352,7 +352,11 @@ def cmd_audit(ws: Workspace, args: argparse.Namespace) -> int: def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: """Check every receipt against the published key and the published schema.""" - receipts = ws.all_receipts() + try: + receipts = ws.all_receipts() + except ReceiptReadError as error: + print(f" {error} -> BAD") + return EXIT_REFUSED if not receipts: print("no receipts to verify", file=sys.stderr) return EXIT_REFUSED diff --git a/prototype/workspace.py b/prototype/workspace.py index c247065..9814982 100644 --- a/prototype/workspace.py +++ b/prototype/workspace.py @@ -25,6 +25,10 @@ from prototype.signing import Ed25519Signer, VerificationKey +class ReceiptReadError(ValueError): + """A persisted receipt cannot be parsed as JSON evidence.""" + + class Workspace: def __init__(self, root: Path) -> None: self.root = Path(root) @@ -238,10 +242,16 @@ def _atomic_json(self, path: Path, payload: object) -> None: def all_receipts(self) -> list[tuple[str, dict[str, Any]]]: if not self.receipts_dir.is_dir(): return [] - return [ - (p.name, json.loads(p.read_text(encoding="utf-8"))) - for p in sorted(self.receipts_dir.glob("*.json")) - ] + receipts = [] + for path in sorted(self.receipts_dir.glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ReceiptReadError( + f"{path.name}: receipt JSON is unreadable" + ) from error + receipts.append((path.name, payload)) + return receipts def latest_receipt(self) -> dict[str, Any] | None: receipts = self.all_receipts() From 4ab1a0fe4046f3717c8fbf86ea04ae817807a1d6 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:15:00 +0200 Subject: [PATCH 20/27] ci: fetch immutable conformance history --- .github/workflows/validate.yml | 4 ++++ scripts/validate_repo.py | 10 ++++++++++ tests/test_validate_repo.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index f8abdd1..c4c95c6 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -24,6 +24,10 @@ jobs: steps: - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1, Node 24 + with: + # Conformance binds historical immutable normative commits. A shallow + # checkout cannot verify those Git objects and must not run the suite. + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0, Node 24 diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index b928f93..7a6fd5d 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -394,6 +394,16 @@ def check_github_actions_runtime(reporter: Reporter) -> None: "Restore immutable Node 24 action pins. " + "; ".join(violations), ) + validate_workflow = ROOT / ".github" / "workflows" / "validate.yml" + validate_text = read_utf8(validate_workflow) if validate_workflow.is_file() else "" + reporter.check( + "GitHub Actions source history", + validate_text.count("fetch-depth: 0") == 1, + "validation checkout fetches immutable Git history required by conformance", + "Set actions/checkout fetch-depth to 0 in validate.yml so historical " + "normative commits are available to fail-closed conformance checks.", + ) + def check_publication_metadata(reporter: Reporter) -> None: readme_path = ROOT / "README.md" diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 4b0d286..12af69d 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -479,6 +479,12 @@ def test_workflows_pin_node24_action_releases(self) -> None: self.assertIn("actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", workflows) self.assertIn("actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", workflows) + def test_validate_workflow_fetches_immutable_history_for_conformance(self) -> None: + workflow = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "validate.yml" + ).read_text(encoding="utf-8") + self.assertIn("fetch-depth: 0", workflow) + def test_validator_rejects_deprecated_action_major(self) -> None: def mutate(root: Path) -> None: path = root / ".github" / "workflows" / "links.yml" @@ -496,6 +502,14 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("GitHub Actions Node 24 pins", result.stdout) + def test_validator_rejects_shallow_validate_checkout(self) -> None: + def mutate(root: Path) -> None: + rewrite(root / ".github" / "workflows" / "validate.yml", "fetch-depth: 0", "fetch-depth: 1") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("GitHub Actions source history", result.stdout) + if __name__ == "__main__": unittest.main() From 1fa500c4fdbafd4098644ee00d9b9378c474bf81 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:18:22 +0200 Subject: [PATCH 21/27] test: bind history guard to checkout step --- scripts/validate_repo.py | 10 +++++++++- tests/test_validate_repo.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 7a6fd5d..db53930 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -396,9 +396,17 @@ def check_github_actions_runtime(reporter: Reporter) -> None: validate_workflow = ROOT / ".github" / "workflows" / "validate.yml" validate_text = read_utf8(validate_workflow) if validate_workflow.is_file() else "" + checkout_step = re.search( + rf"(?ms)^\s{{6}}- name: Check out the repository\s*$\n" + rf"^\s{{8}}uses: actions/checkout@{GITHUB_ACTION_PINS['actions/checkout']}[^\n]*$\n" + rf"^\s{{8}}with:\s*$\n" + rf"(?:(?:^\s{{10}}#.*$\n)*)" + rf"^\s{{10}}fetch-depth:\s*0\s*$", + validate_text, + ) reporter.check( "GitHub Actions source history", - validate_text.count("fetch-depth: 0") == 1, + checkout_step is not None, "validation checkout fetches immutable Git history required by conformance", "Set actions/checkout fetch-depth to 0 in validate.yml so historical " "normative commits are available to fail-closed conformance checks.", diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 12af69d..73376ab 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -510,6 +510,18 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("GitHub Actions source history", result.stdout) + def test_history_guard_ignores_comment_camouflage(self) -> None: + def mutate(root: Path) -> None: + path = root / ".github" / "workflows" / "validate.yml" + text = path.read_text(encoding="utf-8") + text = text.replace("fetch-depth: 0", "fetch-depth: 1", 1) + text += "\n# legacy requirement text: fetch-depth: 0\n" + path.write_text(text, encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("GitHub Actions source history", result.stdout) + if __name__ == "__main__": unittest.main() From 1aa9db88dd442623d172b31de622cfbcc73e5c53 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:32:15 +0200 Subject: [PATCH 22/27] fix: close conformance review findings --- prototype/cli.py | 6 ++- prototype/conformance.py | 75 ++++++++++++++++++++++++++-------- scripts/check_readiness.py | 13 ++++++ spec/README.md | 5 +++ tests/test_cli.py | 7 ++++ tests/test_conformance.py | 83 ++++++++++++++++++++++++++++++++++++++ tests/test_readiness.py | 1 + 7 files changed, 173 insertions(+), 17 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index b22293e..81c851e 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -77,7 +77,10 @@ def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: try: report = validate_adapter_conformance( - args.corpus, args.source_root, _binding_factory(args.binding) + args.corpus, + args.source_root, + _binding_factory(args.binding), + binding_root=args.binding_root, ) except ConformanceInputError as error: print(f"invalid conformance evidence: {error}", file=sys.stderr) @@ -476,6 +479,7 @@ def build_parser() -> argparse.ArgumentParser: ) adapter_conformance.add_argument("--corpus", required=True, type=Path) adapter_conformance.add_argument("--source-root", required=True, type=Path) + adapter_conformance.add_argument("--binding-root", type=Path) adapter_conformance.add_argument("--binding") return parser diff --git a/prototype/conformance.py b/prototype/conformance.py index 9d3f7c7..92cec66 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -29,6 +29,11 @@ def __init__(self, target: object) -> None: def calls(self) -> tuple[str, ...]: return tuple(object.__getattribute__(self, "_calls")) + def reset_calls(self) -> None: + """Start a new trace window around controller-issued lifecycle calls.""" + + object.__getattribute__(self, "_calls").clear() + def __getattr__(self, name: str) -> Any: value = getattr(self.target, name) if not callable(value): @@ -64,7 +69,7 @@ def compare(want: object, got: object, path: str) -> None: child = f"{path}.{key}" if path else key failures.append(f"{child}: unexpected") return - if want != got: + if type(want) is not type(got) or want != got: failures.append(f"{path}: expected {want!r}, got {got!r}") compare(expected, observed, "") @@ -131,6 +136,7 @@ class AdapterCorpus: validator_controls: tuple[dict[str, str], ...] status: str evidence_boundary: str + sha256: str @dataclass(frozen=True) @@ -191,6 +197,7 @@ class ConformanceReport: binding: str normative_commit: str normative_surface_digest: str + corpus_sha256: str runtime_commit: str | None runtime_tree: str binding_source: dict[str, str] @@ -214,6 +221,7 @@ def to_dict(self) -> dict[str, Any]: "commit": self.normative_commit, "surface_digest": self.normative_surface_digest, }, + "corpus_sha256": self.corpus_sha256, "runtime_commit": self.runtime_commit, "runtime_tree": self.runtime_tree, "binding_source": dict(sorted(self.binding_source.items())), @@ -356,9 +364,16 @@ def _validate_outcome(value: object, operation: str, label: str) -> dict[str, An return outcome -def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: +def load_corpus( + path: Path, source_root: Path, *, require_canonical_path: bool = False +) -> AdapterCorpus: + if require_canonical_path and path.resolve() != ( + source_root / "spec" / "adapter-conformance.json" + ).resolve(): + raise ConformanceInputError("corpus is not the canonical checked-in corpus") try: - payload = json.loads(path.read_text(encoding="utf-8")) + corpus_bytes = path.read_bytes() + payload = json.loads(corpus_bytes.decode("utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: raise ConformanceInputError("corpus is not readable canonical JSON") from error root = _exact(payload, ROOT_KEYS, "corpus") @@ -392,6 +407,12 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: raise ConformanceInputError(f"{case_id} operation is invalid") normative = _exact(case["normative"], NORMATIVE_KEYS, f"{case_id} normative") commit = _nonempty(normative["commit"], f"{case_id} normative commit") + try: + _git(source_root, "merge-base", "--is-ancestor", commit, "HEAD") + except ConformanceInputError as error: + raise ConformanceInputError( + f"{case_id} normative commit is not reachable from runtime history" + ) from error relative = _nonempty(normative["path"], f"{case_id} normative path") quote = _nonempty(normative["quote"], f"{case_id} normative quotation") source = _git(source_root, "show", f"{commit}:{relative}").decode("utf-8") @@ -429,6 +450,7 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: validator_controls=tuple(controls), status=_nonempty(root["status"], "status"), evidence_boundary=_nonempty(root["evidence_boundary"], "evidence boundary"), + sha256=hashlib.sha256(corpus_bytes).hexdigest(), ) @@ -748,9 +770,12 @@ def _run_case( if mutate: binding.apply_mutation(case, importer, adapter, context) erratum = binding.erratum(case, context) + adapter.reset_calls() checkpoint = importer.quarantine(erratum) receipt = importer.repair_quarantined(erratum, checkpoint) - return binding.observe(case, importer, adapter, context, checkpoint, receipt), adapter.calls + calls = adapter.calls + observed = binding.observe(case, importer, adapter, context, checkpoint, receipt) + return observed, calls def _anti_vacuity_receipt( @@ -766,7 +791,7 @@ def _anti_vacuity_receipt( def _anti_vacuity_feed( corpus: AdapterCorpus, feed_verifier: Callable[..., list[object]] ) -> str: - from prototype.errata import Erratum, Operation, RootRegistry + from prototype.errata import Erratum, FeedError, Operation, RootRegistry from prototype.signing import Ed25519Signer mutation = next( @@ -791,9 +816,15 @@ def _anti_vacuity_feed( ) try: feed_verifier([event], owner=owner.public, roots=RootRegistry({"fact:diet"})) - except Exception as error: - message = str(error) - return mutation["required_failure"] if "gap" in message else "" + except FeedError as error: + expected = ( + "anti-vacuity-gap: gap at sequence 2, expected 1. " + "A missing erratum may be the one that retired the state this importer " + "is about to serve." + ) + return mutation["required_failure"] if str(error) == expected else "" + except Exception: + return "" return "" @@ -879,24 +910,34 @@ def _runtime_identity(root: Path) -> tuple[str, str]: def _binding_source( - source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding] + binding_factory: Callable[[], ReferenceConformanceBinding], + binding_root: Path | None, ) -> dict[str, str]: try: file_path = Path(inspect.getsourcefile(binding_factory) or "").resolve() - relative = file_path.relative_to(source_root.resolve()).as_posix() + root = binding_root.resolve() if binding_root is not None else Path( + _git(file_path.parent, "rev-parse", "--show-toplevel").decode().strip() + ).resolve() + relative = file_path.relative_to(root).as_posix() payload = file_path.read_bytes() - except (OSError, ValueError) as error: + except (OSError, TypeError, ValueError) as error: raise ConformanceInputError( - "binding source is not inside the declared source root" + "binding source is not inside a clean Git repository" ) from error if relative.startswith(".git/"): raise ConformanceInputError("binding source is not executable repository source") tracked = set( - _git(source_root, "ls-files", "--cached").decode("utf-8").splitlines() + _git(root, "ls-files", "--cached").decode("utf-8").splitlines() ) if relative not in tracked: raise ConformanceInputError("binding source is not tracked by the runtime tree") - return {"path": relative, "sha256": hashlib.sha256(payload).hexdigest()} + commit, tree = _runtime_identity(root) + return { + "commit": commit, + "tree": tree, + "path": relative, + "sha256": hashlib.sha256(payload).hexdigest(), + } class _BindingTimeout(Exception): @@ -926,10 +967,11 @@ def validate_adapter_conformance( corpus_path: Path, source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding], + binding_root: Path | None = None, ) -> ConformanceReport: runtime_commit, runtime_tree = _runtime_identity(source_root) - corpus = load_corpus(corpus_path, source_root) - binding_source = _binding_source(source_root, binding_factory) + corpus = load_corpus(corpus_path, source_root, require_canonical_path=True) + binding_source = _binding_source(binding_factory, binding_root) try: binding = _run_with_timeout(binding_factory) except _BindingTimeout as error: @@ -1001,6 +1043,7 @@ def validate_adapter_conformance( binding=binding_name, normative_commit=corpus.normative_target.commit, normative_surface_digest=corpus.normative_target.surface_digest, + corpus_sha256=corpus.sha256, runtime_commit=runtime_commit, runtime_tree=runtime_tree, binding_source=binding_source, diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 2cfb77d..0464da0 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -188,6 +188,7 @@ def g2_surface_files(root: Path = ROOT) -> tuple[str, ...]: tuple(sorted((root / "prototype").glob("*.py"))), (root / "prototype" / "README.md",), (root / "spec" / "README.md",), + (root / "spec" / "adapter-conformance.json",), tuple(sorted((root / "spec").glob("*.schema.json"))), tuple(sorted((root / "spec" / "vectors").glob("*.json"))), tuple(sorted((root / "spec" / "semantic").glob("*.json"))), @@ -222,8 +223,20 @@ def g2_surface_digest(root: Path = ROOT) -> str: def g2_surface_digest_at_commit(commit: str, root: Path = ROOT) -> str: if not reviewed_commit_exists(commit, root): raise OSError("reviewed commit is unavailable") + listed = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", commit], + cwd=root, + capture_output=True, + check=False, + text=True, + ) + if listed.returncode != 0: + raise OSError("reviewed commit tree is unavailable") + commit_files = set(listed.stdout.splitlines()) entries = [] for relative in g2_surface_files(root): + if relative == "spec/adapter-conformance.json" and relative not in commit_files: + continue result = subprocess.run( ["git", "show", f"{commit}:{relative}"], cwd=root, capture_output=True, check=False ) diff --git a/spec/README.md b/spec/README.md index 070fad0..b8afb96 100644 --- a/spec/README.md +++ b/spec/README.md @@ -41,6 +41,11 @@ python3 -m prototype.cli adapter-conformance \ --source-root . ``` +For an independently implemented binding in another checkout, add +`--binding module:factory --binding-root /path/to/its/clean/git/checkout`. +Reports bind the LLM Errata runtime commit/tree, exact corpus SHA-256, and the +binding repository commit/tree plus tracked source path and SHA-256 separately. + Exit `0` means the supplied binding and validator controls passed this internal corpus. Exit `1` means a behavioral or mutation control failed. Exit `2` means source, corpus, binding, or execution evidence is invalid or diff --git a/tests/test_cli.py b/tests/test_cli.py index fe18e95..e2db04d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -90,6 +90,7 @@ def test_reference_binding_emits_canonical_passing_report(self) -> None: self.assertTrue(payload["passed"]) self.assertEqual(len(payload["cases"]), 5) self.assertEqual(len(payload["validator_controls"]), 3) + self.assertRegex(payload["corpus_sha256"], r"^[0-9a-f]{64}$") self.assertIn("not G2 or G4 evidence", payload["evidence_boundary"]) self.assertEqual(result.stdout.strip(), json.dumps( payload, sort_keys=True, separators=(",", ":") @@ -112,6 +113,12 @@ def test_invalid_source_evidence_exits_two(self) -> None: self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) self.assertIn("invalid conformance evidence", result.stderr) + def test_sourceless_binding_factory_exits_two_without_traceback(self) -> None: + result = self.run_conformance("--binding", "builtins:dict") + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertEqual(result.stdout, "") + self.assertNotIn("Traceback", result.stderr) + class WorkspaceLifecycle(CliCase): def test_commands_refuse_before_init(self) -> None: diff --git a/tests/test_conformance.py b/tests/test_conformance.py index dfea682..b200cc9 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -59,6 +59,20 @@ def test_wrong_surface_digest_is_refused(self) -> None: with self.assertRaisesRegex(ConformanceInputError, "surface digest"): load_corpus(path, ROOT) + def test_noncanonical_corpus_path_is_refused_for_execution(self) -> None: + path = self.changed_corpus(lambda value: None) + with self.assertRaisesRegex(ConformanceInputError, "canonical checked-in"): + load_corpus(path, ROOT, require_canonical_path=True) + + def test_normative_source_must_be_reachable_from_runtime_history(self) -> None: + path = self.changed_corpus( + lambda value: value["cases"][0]["normative"].__setitem__( + "commit", "f" * 40 + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "reachable"): + load_corpus(path, ROOT) + def test_new_current_surface_files_do_not_change_historical_manifest(self) -> None: corpus = load_corpus(CORPUS, ROOT) self.assertEqual( @@ -139,6 +153,20 @@ class Adapter: self.assertEqual(traced.name, "target") self.assertEqual(traced.calls, ()) + def test_trace_window_can_exclude_fixture_and_observation_calls(self) -> None: + class Adapter: + def coverage(self, root: str) -> str: + return root + + traced = TracingAdapter(Adapter()) + traced.coverage("fixture") + traced.reset_calls() + traced.coverage("controller") + calls = traced.calls + traced.coverage("observation") + self.assertEqual(calls, ("coverage",)) + self.assertEqual(traced.calls, ("coverage", "coverage")) + class CompleteComparison(unittest.TestCase): def outcome(self) -> dict[str, object]: @@ -189,6 +217,17 @@ def test_missing_or_extra_fields_are_rejected(self) -> None: self.assertIn("receipt.names_store: missing", failures) self.assertIn("store.unexpected: unexpected", failures) + def test_json_scalar_types_must_match_exactly(self) -> None: + for wrong in (1, 1.0): + with self.subTest(wrong=wrong): + expected = self.outcome() + observed = self.outcome() + observed["receipt"]["names_store"] = wrong + self.assertIn( + f"receipt.names_store: expected True, got {wrong!r}", + compare_complete_outcome(expected, observed), + ) + class PropositionMultiplicity(unittest.TestCase): def test_reference_observations_use_stable_provider_local_ids_and_counts(self) -> None: @@ -292,6 +331,39 @@ def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None self.assertEqual(report.runtime_tree, tree) self.assertEqual(report.binding_source["path"], "prototype/conformance.py") self.assertRegex(report.binding_source["sha256"], r"^[0-9a-f]{64}$") + self.assertEqual(report.binding_source["commit"], commit) + self.assertEqual(report.binding_source["tree"], tree) + self.assertRegex(report.corpus_sha256, r"^[0-9a-f]{64}$") + + def test_external_binding_can_be_bound_to_its_own_clean_repository(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-") as directory: + binding_root = Path(directory) + source = binding_root / "binding.py" + source.write_text("def factory():\n return None\n", encoding="utf-8") + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "binding.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=binding_root, check=True) + namespace: dict[str, object] = {} + exec(compile(source.read_text(), str(source), "exec"), namespace) + factory = namespace["factory"] + factory.__module__ = "binding" + with patch("prototype.conformance.inspect.getsourcefile", return_value=str(source)): + from prototype.conformance import _binding_source + + identity = _binding_source(factory, binding_root) + self.assertEqual(identity["path"], "binding.py") + self.assertRegex(identity["commit"], r"^[0-9a-f]{40}$") + + def test_sourceless_binding_factory_is_invalid_evidence(self) -> None: + from prototype.conformance import _binding_source + + with self.assertRaisesRegex(ConformanceInputError, "clean Git repository"): + _binding_source(dict, None) def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: @@ -366,6 +438,17 @@ def test_no_op_feed_verifier_makes_acceptance_control_fail(self) -> None: ) self.assertFalse(controls[1].passed) + def test_unrelated_gap_exception_cannot_satisfy_feed_control(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + + def crash(*args, **kwargs): + raise RuntimeError("unrelated gap in logging") + + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, feed_verifier=crash, + ) + self.assertFalse(controls[1].passed) + def test_constant_unknown_semantic_runner_makes_diversity_control_fail(self) -> None: from prototype.semantic import SemanticCoverage diff --git a/tests/test_readiness.py b/tests/test_readiness.py index d2c09ff..20177e5 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -89,6 +89,7 @@ def test_g2_surface_includes_checkpoint_contract_and_tests(self) -> None: files = set(g2_surface_files()) self.assertIn("prototype/checkpoints.py", files) self.assertIn("tests/test_checkpoints.py", files) + self.assertIn("spec/adapter-conformance.json", files) def test_g6_complete_measured_report_is_commit_and_deployment_bound(self) -> None: with repo_copy() as source, tempfile.TemporaryDirectory() as temp: From 03d1a23877eb7b5a43bbdd55fac0a9fca7f871be Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:43:05 +0200 Subject: [PATCH 23/27] fix: bound external binding imports --- prototype/cli.py | 29 +++++-------- prototype/conformance.py | 86 ++++++++++++++++++++++++++++++++++++++- tests/test_cli.py | 36 ++++++++++++++-- tests/test_conformance.py | 41 +++++++++++++++++++ 4 files changed, 169 insertions(+), 23 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index 81c851e..ded61e4 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -21,7 +21,6 @@ from __future__ import annotations import argparse -import importlib import json import sys from datetime import datetime, timezone @@ -32,6 +31,7 @@ from prototype.conformance import ( ConformanceInputError, ReferenceConformanceBinding, + load_binding_factory, validate_adapter_conformance, ) from prototype.controller import Importer, Phase @@ -57,30 +57,23 @@ EXIT_INCONCLUSIVE = 2 -def _binding_factory(value: str | None): - if value is None: - return ReferenceConformanceBinding - try: - module_name, object_name = value.split(":", 1) - factory = getattr(importlib.import_module(module_name), object_name) - except (AttributeError, ImportError, ValueError) as error: - raise ConformanceInputError( - "binding must be an importable module:factory" - ) from error - if not callable(factory): - raise ConformanceInputError("binding factory must be callable") - return factory - - def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: """Run provider-neutral adapter cases and validator self-controls.""" try: + if args.binding is None: + factory = ReferenceConformanceBinding + source = None + binding_root = args.binding_root + else: + binding_root = args.binding_root or args.source_root + factory, source = load_binding_factory(args.binding, binding_root) report = validate_adapter_conformance( args.corpus, args.source_root, - _binding_factory(args.binding), - binding_root=args.binding_root, + factory, + binding_root=binding_root, + binding_source=source, ) except ConformanceInputError as error: print(f"invalid conformance evidence: {error}", file=sys.stderr) diff --git a/prototype/conformance.py b/prototype/conformance.py index 92cec66..633504c 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -3,11 +3,13 @@ from __future__ import annotations import hashlib +import importlib import inspect import json import re import signal import subprocess +import sys import threading from dataclasses import dataclass from pathlib import Path @@ -940,6 +942,85 @@ def _binding_source( } +def load_binding_factory( + value: str, + binding_root: Path, +) -> tuple[Callable[[], ReferenceConformanceBinding], dict[str, str]]: + """Admit tracked module bytes before bounded import executes them.""" + + try: + module_name, object_name = value.split(":", 1) + except ValueError as error: + raise ConformanceInputError( + "binding must be an importable module:factory" + ) from error + identifier = re.compile(r"^[A-Za-z_]\w*$") + parts = module_name.split(".") + if not parts or not all(identifier.fullmatch(part) for part in parts) or not identifier.fullmatch( + object_name + ): + raise ConformanceInputError("binding must be an importable module:factory") + + root = binding_root.resolve() + module_path = root.joinpath(*parts).with_suffix(".py") + package_path = root.joinpath(*parts, "__init__.py") + candidates = [path for path in (module_path, package_path) if path.is_file()] + if len(candidates) != 1: + raise ConformanceInputError("binding module must resolve to one repository file") + admitted_path = candidates[0].resolve() + relative = admitted_path.relative_to(root).as_posix() + tracked = set(_git(root, "ls-files", "--cached").decode("utf-8").splitlines()) + if relative not in tracked: + raise ConformanceInputError("binding source is not tracked by the runtime tree") + commit, tree = _runtime_identity(root) + source = { + "commit": commit, + "tree": tree, + "path": relative, + "sha256": hashlib.sha256(admitted_path.read_bytes()).hexdigest(), + } + + try: + search_locations = [str(admitted_path.parent)] if admitted_path.name == "__init__.py" else None + spec = importlib.util.spec_from_file_location( + module_name, admitted_path, submodule_search_locations=search_locations + ) + if spec is None or spec.loader is None: + raise ImportError("binding module loader is unavailable") + module = importlib.util.module_from_spec(spec) + prior_module = sys.modules.get(module_name) + sys.modules[module_name] = module + + def execute_import() -> None: + spec.loader.exec_module(module) + + _run_with_timeout(execute_import) + factory = getattr(module, object_name) + except _BindingTimeout as error: + raise ConformanceInputError("binding import timed out") from error + except BaseException as error: + if isinstance(error, KeyboardInterrupt): + raise + raise ConformanceInputError( + f"binding import failed: {type(error).__name__}" + ) from error + finally: + if "prior_module" in locals(): + if prior_module is None: + sys.modules.pop(module_name, None) + else: + sys.modules[module_name] = prior_module + if not callable(factory): + raise ConformanceInputError("binding factory must be callable") + try: + loaded_path = Path(inspect.getsourcefile(factory) or "").resolve() + except (OSError, TypeError, ValueError) as error: + raise ConformanceInputError("binding factory has no admitted source") from error + if loaded_path != admitted_path: + raise ConformanceInputError("binding factory source differs from admitted module") + return factory, source + + class _BindingTimeout(Exception): pass @@ -968,10 +1049,11 @@ def validate_adapter_conformance( source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding], binding_root: Path | None = None, + binding_source: dict[str, str] | None = None, ) -> ConformanceReport: runtime_commit, runtime_tree = _runtime_identity(source_root) corpus = load_corpus(corpus_path, source_root, require_canonical_path=True) - binding_source = _binding_source(binding_factory, binding_root) + source_identity = binding_source or _binding_source(binding_factory, binding_root) try: binding = _run_with_timeout(binding_factory) except _BindingTimeout as error: @@ -1046,7 +1128,7 @@ def validate_adapter_conformance( corpus_sha256=corpus.sha256, runtime_commit=runtime_commit, runtime_tree=runtime_tree, - binding_source=binding_source, + binding_source=source_identity, cases=tuple(results), validator_controls=run_validator_anti_vacuity_controls(corpus, source_root), provenance=corpus.provenance, diff --git a/tests/test_cli.py b/tests/test_cli.py index e2db04d..87d62e4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,14 +41,19 @@ def setUp(self) -> None: def tearDown(self) -> None: self._tmp.cleanup() - def run_cli(self, *args: str) -> subprocess.CompletedProcess[str]: + def run_cli( + self, *args: str, extra_pythonpath: Path | None = None + ) -> subprocess.CompletedProcess[str]: + pythonpath = str(REPO_ROOT) + if extra_pythonpath is not None: + pythonpath = f"{extra_pythonpath}:{pythonpath}" return subprocess.run( [sys.executable, "-m", "prototype.cli", "--workspace", ".errata", *args], cwd=self.cwd, capture_output=True, text=True, check=False, - env={"PYTHONPATH": str(REPO_ROOT), "PATH": "/usr/bin:/bin"}, + env={"PYTHONPATH": pythonpath, "PATH": "/usr/bin:/bin"}, ) def seed(self) -> None: @@ -75,12 +80,15 @@ def quarantine_and_repair(self) -> subprocess.CompletedProcess[str]: class AdapterConformanceCommand(CliCase): - def run_conformance(self, *extra: str) -> subprocess.CompletedProcess[str]: + def run_conformance( + self, *extra: str, extra_pythonpath: Path | None = None + ) -> subprocess.CompletedProcess[str]: return self.run_cli( "adapter-conformance", "--corpus", str(REPO_ROOT / "spec" / "adapter-conformance.json"), "--source-root", str(REPO_ROOT), *extra, + extra_pythonpath=extra_pythonpath, ) def test_reference_binding_emits_canonical_passing_report(self) -> None: @@ -119,6 +127,28 @@ def test_sourceless_binding_factory_exits_two_without_traceback(self) -> None: self.assertEqual(result.stdout, "") self.assertNotIn("Traceback", result.stderr) + def test_crashing_binding_import_exits_two_without_traceback(self) -> None: + binding_root = self.cwd / "binding" + binding_root.mkdir() + (binding_root / "broken.py").write_text( + 'raise RuntimeError("top-level failure")\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "CLI Tests"), + ("git", "add", "broken.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=binding_root, check=True) + result = self.run_conformance( + "--binding", "broken:factory", "--binding-root", str(binding_root), + extra_pythonpath=binding_root, + ) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertIn("binding import failed: RuntimeError", result.stderr) + self.assertNotIn("Traceback", result.stderr) + class WorkspaceLifecycle(CliCase): def test_commands_refuse_before_init(self) -> None: diff --git a/tests/test_conformance.py b/tests/test_conformance.py index b200cc9..597bae8 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -8,6 +8,7 @@ import json import subprocess +import sys import tempfile import time import unittest @@ -23,6 +24,7 @@ compare_complete_outcome, compare_proposition_multiplicity, load_corpus, + load_binding_factory, run_validator_anti_vacuity_controls, validate_adapter_conformance, ) @@ -365,6 +367,45 @@ def test_sourceless_binding_factory_is_invalid_evidence(self) -> None: with self.assertRaisesRegex(ConformanceInputError, "clean Git repository"): _binding_source(dict, None) + def test_binding_import_runtime_error_is_invalid_evidence(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + (root / "broken.py").write_text( + 'raise RuntimeError("top-level failure")\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "broken.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + with patch("sys.path", [str(root), *sys.path]): + with self.assertRaisesRegex(ConformanceInputError, "RuntimeError"): + load_binding_factory("broken:factory", root) + + def test_binding_import_has_a_hard_timeout(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + (root / "slow.py").write_text( + "import time\ntime.sleep(0.1)\ndef factory():\n return None\n", + encoding="utf-8", + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "slow.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + with patch("sys.path", [str(root), *sys.path]), patch( + "prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01 + ): + with self.assertRaisesRegex(ConformanceInputError, "import timed out"): + load_binding_factory("slow:factory", root) + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From 41fb070cd8baccd2db8a6aaacabd4d6886e66bfe Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:47:47 +0200 Subject: [PATCH 24/27] fix: execute admitted binding bytes --- prototype/conformance.py | 21 +++++++++++++++------ tests/test_conformance.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 633504c..da3a596 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -973,26 +973,32 @@ def load_binding_factory( if relative not in tracked: raise ConformanceInputError("binding source is not tracked by the runtime tree") commit, tree = _runtime_identity(root) + admitted_bytes = admitted_path.read_bytes() source = { "commit": commit, "tree": tree, "path": relative, - "sha256": hashlib.sha256(admitted_path.read_bytes()).hexdigest(), + "sha256": hashlib.sha256(admitted_bytes).hexdigest(), } try: search_locations = [str(admitted_path.parent)] if admitted_path.name == "__init__.py" else None - spec = importlib.util.spec_from_file_location( - module_name, admitted_path, submodule_search_locations=search_locations + spec = importlib.util.spec_from_loader( + module_name, loader=None, origin=str(admitted_path), + is_package=search_locations is not None, ) - if spec is None or spec.loader is None: - raise ImportError("binding module loader is unavailable") + if spec is None: + raise ImportError("binding module specification is unavailable") + if search_locations is not None: + spec.submodule_search_locations = search_locations module = importlib.util.module_from_spec(spec) + module.__file__ = str(admitted_path) prior_module = sys.modules.get(module_name) sys.modules[module_name] = module def execute_import() -> None: - spec.loader.exec_module(module) + code = compile(admitted_bytes, str(admitted_path), "exec") + exec(code, module.__dict__) _run_with_timeout(execute_import) factory = getattr(module, object_name) @@ -1018,6 +1024,9 @@ def execute_import() -> None: raise ConformanceInputError("binding factory has no admitted source") from error if loaded_path != admitted_path: raise ConformanceInputError("binding factory source differs from admitted module") + post_commit, post_tree = _runtime_identity(root) + if post_commit != commit or post_tree != tree: + raise ConformanceInputError("binding repository identity changed during import") return factory, source diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 597bae8..9e85713 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -406,6 +406,35 @@ def test_binding_import_has_a_hard_timeout(self) -> None: with self.assertRaisesRegex(ConformanceInputError, "import timed out"): load_binding_factory("slow:factory", root) + def test_binding_executes_the_admitted_bytes_not_a_reopened_path(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + source = root / "stable.py" + source.write_text( + 'def factory():\n return "ADMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "stable.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + original_read = Path.read_bytes + + def substitute_after_read(path: Path) -> bytes: + payload = original_read(path) + if path.resolve() == source.resolve(): + source.write_text( + 'def factory():\n return "SUBSTITUTED"\n', encoding="utf-8" + ) + return payload + + with patch("pathlib.Path.read_bytes", substitute_after_read): + with self.assertRaisesRegex(ConformanceInputError, "dirty"): + load_binding_factory("stable:factory", root) + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From 08e3f5909030fd9713aadb729a8c89599cd06a14 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:53:22 +0200 Subject: [PATCH 25/27] fix: isolate binding package imports --- prototype/conformance.py | 94 +++++++++++++++++++++++++++++---------- spec/README.md | 2 + tests/test_conformance.py | 39 ++++++++++++++++ 3 files changed, 112 insertions(+), 23 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index da3a596..5ea00b9 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -4,6 +4,8 @@ import hashlib import importlib +import importlib.abc +import importlib.util import inspect import json import re @@ -981,26 +983,62 @@ def load_binding_factory( "sha256": hashlib.sha256(admitted_bytes).hexdigest(), } - try: - search_locations = [str(admitted_path.parent)] if admitted_path.name == "__init__.py" else None - spec = importlib.util.spec_from_loader( - module_name, loader=None, origin=str(admitted_path), - is_package=search_locations is not None, - ) - if spec is None: - raise ImportError("binding module specification is unavailable") - if search_locations is not None: - spec.submodule_search_locations = search_locations - module = importlib.util.module_from_spec(spec) - module.__file__ = str(admitted_path) - prior_module = sys.modules.get(module_name) - sys.modules[module_name] = module - - def execute_import() -> None: - code = compile(admitted_bytes, str(admitted_path), "exec") + executed_sources: dict[str, tuple[str, bytes]] = {} + + class AdmittedLoader(importlib.abc.Loader): + def __init__(self, fullname: str, path: Path, payload: bytes) -> None: + self.fullname = fullname + self.path = path + self.payload = payload + + def create_module(self, spec: object) -> None: + return None + + def exec_module(self, module: object) -> None: + relative_path = self.path.relative_to(root).as_posix() + executed_sources[self.fullname] = (relative_path, self.payload) + code = compile(self.payload, str(self.path), "exec") exec(code, module.__dict__) - _run_with_timeout(execute_import) + class AdmittedFinder(importlib.abc.MetaPathFinder): + def find_spec( + self, fullname: str, path: object = None, target: object = None + ) -> object: + if fullname != module_name and not fullname.startswith(f"{module_name}."): + return None + name_parts = fullname.split(".") + source_path = root.joinpath(*name_parts).with_suffix(".py") + init_path = root.joinpath(*name_parts, "__init__.py") + matches = [item.resolve() for item in (source_path, init_path) if item.is_file()] + if len(matches) != 1: + raise ImportError(f"binding dependency {fullname!r} is ambiguous or missing") + selected = matches[0] + selected_relative = selected.relative_to(root).as_posix() + if selected_relative not in tracked: + raise ImportError(f"binding dependency {fullname!r} is not tracked") + payload = admitted_bytes if fullname == module_name else selected.read_bytes() + loader = AdmittedLoader(fullname, selected, payload) + is_package = selected.name == "__init__.py" + return importlib.util.spec_from_file_location( + fullname, + selected, + loader=loader, + submodule_search_locations=[str(selected.parent)] if is_package else None, + ) + + namespace_prefix = f"{module_name}." + saved_modules = { + name: module + for name, module in tuple(sys.modules.items()) + if name == module_name or name.startswith(namespace_prefix) + } + for name in saved_modules: + sys.modules.pop(name, None) + finder = AdmittedFinder() + sys.meta_path.insert(0, finder) + + try: + module = _run_with_timeout(lambda: importlib.import_module(module_name)) factory = getattr(module, object_name) except _BindingTimeout as error: raise ConformanceInputError("binding import timed out") from error @@ -1011,11 +1049,12 @@ def execute_import() -> None: f"binding import failed: {type(error).__name__}" ) from error finally: - if "prior_module" in locals(): - if prior_module is None: - sys.modules.pop(module_name, None) - else: - sys.modules[module_name] = prior_module + if finder in sys.meta_path: + sys.meta_path.remove(finder) + for name in tuple(sys.modules): + if name == module_name or name.startswith(namespace_prefix): + sys.modules.pop(name, None) + sys.modules.update(saved_modules) if not callable(factory): raise ConformanceInputError("binding factory must be callable") try: @@ -1027,6 +1066,15 @@ def execute_import() -> None: post_commit, post_tree = _runtime_identity(root) if post_commit != commit or post_tree != tree: raise ConformanceInputError("binding repository identity changed during import") + dependency_digest = hashlib.sha256() + for fullname, (dependency_path, payload) in sorted(executed_sources.items()): + dependency_digest.update(fullname.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(dependency_path.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(hashlib.sha256(payload).digest()) + dependency_digest.update(b"\0") + source["dependency_manifest_sha256"] = dependency_digest.hexdigest() return factory, source diff --git a/spec/README.md b/spec/README.md index b8afb96..83c013a 100644 --- a/spec/README.md +++ b/spec/README.md @@ -45,6 +45,8 @@ For an independently implemented binding in another checkout, add `--binding module:factory --binding-root /path/to/its/clean/git/checkout`. Reports bind the LLM Errata runtime commit/tree, exact corpus SHA-256, and the binding repository commit/tree plus tracked source path and SHA-256 separately. +Binding-package modules are loaded from admitted tracked bytes rather than any +preloaded module cache, and the report binds their dependency-manifest digest. Exit `0` means the supplied binding and validator controls passed this internal corpus. Exit `1` means a behavioral or mutation control failed. Exit diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 9e85713..b302ba1 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -11,6 +11,7 @@ import sys import tempfile import time +import types import unittest from pathlib import Path from unittest.mock import patch @@ -435,6 +436,44 @@ def substitute_after_read(path: Path) -> bytes: with self.assertRaisesRegex(ConformanceInputError, "dirty"): load_binding_factory("stable:factory", root) + def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text( + "from .helper import VALUE\n" + "def factory():\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached = types.ModuleType("bindingpkg.helper") + cached.VALUE = "CACHED-UNBOUND" + prior = sys.modules.get("bindingpkg.helper") + sys.modules["bindingpkg.helper"] = cached + try: + factory, identity = load_binding_factory("bindingpkg:factory", root) + finally: + if prior is None: + sys.modules.pop("bindingpkg.helper", None) + else: + sys.modules["bindingpkg.helper"] = prior + self.assertEqual(factory(), "COMMITTED") + self.assertRegex( + identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" + ) + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From c7fa670e77b6b42314e0f25f5d0921041ac9eaa0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:56:45 +0200 Subject: [PATCH 26/27] fix: isolate dotted binding namespaces --- prototype/conformance.py | 12 +++++---- tests/test_conformance.py | 51 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 5ea00b9..5dc716d 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -1000,11 +1000,13 @@ def exec_module(self, module: object) -> None: code = compile(self.payload, str(self.path), "exec") exec(code, module.__dict__) + top_package = parts[0] + class AdmittedFinder(importlib.abc.MetaPathFinder): def find_spec( self, fullname: str, path: object = None, target: object = None ) -> object: - if fullname != module_name and not fullname.startswith(f"{module_name}."): + if fullname != top_package and not fullname.startswith(f"{top_package}."): return None name_parts = fullname.split(".") source_path = root.joinpath(*name_parts).with_suffix(".py") @@ -1016,7 +1018,7 @@ def find_spec( selected_relative = selected.relative_to(root).as_posix() if selected_relative not in tracked: raise ImportError(f"binding dependency {fullname!r} is not tracked") - payload = admitted_bytes if fullname == module_name else selected.read_bytes() + payload = admitted_bytes if selected == admitted_path else selected.read_bytes() loader = AdmittedLoader(fullname, selected, payload) is_package = selected.name == "__init__.py" return importlib.util.spec_from_file_location( @@ -1026,11 +1028,11 @@ def find_spec( submodule_search_locations=[str(selected.parent)] if is_package else None, ) - namespace_prefix = f"{module_name}." + namespace_prefix = f"{top_package}." saved_modules = { name: module for name, module in tuple(sys.modules.items()) - if name == module_name or name.startswith(namespace_prefix) + if name == top_package or name.startswith(namespace_prefix) } for name in saved_modules: sys.modules.pop(name, None) @@ -1052,7 +1054,7 @@ def find_spec( if finder in sys.meta_path: sys.meta_path.remove(finder) for name in tuple(sys.modules): - if name == module_name or name.startswith(namespace_prefix): + if name == top_package or name.startswith(namespace_prefix): sys.modules.pop(name, None) sys.modules.update(saved_modules) if not callable(factory): diff --git a/tests/test_conformance.py b/tests/test_conformance.py index b302ba1..5f86237 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -474,6 +474,57 @@ def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> No identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" ) + def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "subbinding.py").write_text( + "from .helper import VALUE\n" + "def factory():\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached_parent = types.ModuleType("bindingpkg") + cached_parent.__path__ = [str(package)] + cached_helper = types.ModuleType("bindingpkg.helper") + cached_helper.VALUE = "CACHED-UNBOUND" + prior_parent = sys.modules.get("bindingpkg") + prior_helper = sys.modules.get("bindingpkg.helper") + sys.modules["bindingpkg"] = cached_parent + sys.modules["bindingpkg.helper"] = cached_helper + try: + factory, identity = load_binding_factory( + "bindingpkg.subbinding:factory", root + ) + self.assertIs(sys.modules["bindingpkg"], cached_parent) + self.assertIs(sys.modules["bindingpkg.helper"], cached_helper) + finally: + if prior_parent is None: + sys.modules.pop("bindingpkg", None) + else: + sys.modules["bindingpkg"] = prior_parent + if prior_helper is None: + sys.modules.pop("bindingpkg.helper", None) + else: + sys.modules["bindingpkg.helper"] = prior_helper + self.assertEqual(factory(), "COMMITTED") + self.assertRegex( + identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" + ) + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From d7f5a451613c7f8bb40eba9f3e63e16255d8b494 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 01:03:38 +0200 Subject: [PATCH 27/27] fix: isolate binding imports for full run --- prototype/cli.py | 24 ++++++++----- prototype/conformance.py | 71 +++++++++++++++++++++------------------ tests/test_conformance.py | 70 ++++++++++++++++++++++++++++++++------ 3 files changed, 113 insertions(+), 52 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index ded61e4..ff0a596 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -67,14 +67,22 @@ def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: binding_root = args.binding_root else: binding_root = args.binding_root or args.source_root - factory, source = load_binding_factory(args.binding, binding_root) - report = validate_adapter_conformance( - args.corpus, - args.source_root, - factory, - binding_root=binding_root, - binding_source=source, - ) + with load_binding_factory(args.binding, binding_root) as (factory, source): + report = validate_adapter_conformance( + args.corpus, + args.source_root, + factory, + binding_root=binding_root, + binding_source=source, + ) + if args.binding is None: + report = validate_adapter_conformance( + args.corpus, + args.source_root, + factory, + binding_root=binding_root, + binding_source=source, + ) except ConformanceInputError as error: print(f"invalid conformance evidence: {error}", file=sys.stderr) return EXIT_INCONCLUSIVE diff --git a/prototype/conformance.py b/prototype/conformance.py index 5dc716d..4d03190 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -13,6 +13,7 @@ import subprocess import sys import threading +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any, Callable @@ -944,11 +945,12 @@ def _binding_source( } +@contextmanager def load_binding_factory( value: str, binding_root: Path, -) -> tuple[Callable[[], ReferenceConformanceBinding], dict[str, str]]: - """Admit tracked module bytes before bounded import executes them.""" +) -> Any: + """Keep an admitted package namespace isolated for the complete run.""" try: module_name, object_name = value.split(":", 1) @@ -1040,16 +1042,40 @@ def find_spec( sys.meta_path.insert(0, finder) try: - module = _run_with_timeout(lambda: importlib.import_module(module_name)) - factory = getattr(module, object_name) - except _BindingTimeout as error: - raise ConformanceInputError("binding import timed out") from error - except BaseException as error: - if isinstance(error, KeyboardInterrupt): - raise - raise ConformanceInputError( - f"binding import failed: {type(error).__name__}" - ) from error + try: + module = _run_with_timeout(lambda: importlib.import_module(module_name)) + factory = getattr(module, object_name) + except _BindingTimeout as error: + raise ConformanceInputError("binding import timed out") from error + except BaseException as error: + if isinstance(error, KeyboardInterrupt): + raise + raise ConformanceInputError( + f"binding import failed: {type(error).__name__}" + ) from error + if not callable(factory): + raise ConformanceInputError("binding factory must be callable") + try: + loaded_path = Path(inspect.getsourcefile(factory) or "").resolve() + except (OSError, TypeError, ValueError) as error: + raise ConformanceInputError("binding factory has no admitted source") from error + if loaded_path != admitted_path: + raise ConformanceInputError("binding factory source differs from admitted module") + + yield factory, source + + post_commit, post_tree = _runtime_identity(root) + if post_commit != commit or post_tree != tree: + raise ConformanceInputError("binding repository identity changed during execution") + dependency_digest = hashlib.sha256() + for fullname, (dependency_path, payload) in sorted(executed_sources.items()): + dependency_digest.update(fullname.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(dependency_path.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(hashlib.sha256(payload).digest()) + dependency_digest.update(b"\0") + source["dependency_manifest_sha256"] = dependency_digest.hexdigest() finally: if finder in sys.meta_path: sys.meta_path.remove(finder) @@ -1057,27 +1083,6 @@ def find_spec( if name == top_package or name.startswith(namespace_prefix): sys.modules.pop(name, None) sys.modules.update(saved_modules) - if not callable(factory): - raise ConformanceInputError("binding factory must be callable") - try: - loaded_path = Path(inspect.getsourcefile(factory) or "").resolve() - except (OSError, TypeError, ValueError) as error: - raise ConformanceInputError("binding factory has no admitted source") from error - if loaded_path != admitted_path: - raise ConformanceInputError("binding factory source differs from admitted module") - post_commit, post_tree = _runtime_identity(root) - if post_commit != commit or post_tree != tree: - raise ConformanceInputError("binding repository identity changed during import") - dependency_digest = hashlib.sha256() - for fullname, (dependency_path, payload) in sorted(executed_sources.items()): - dependency_digest.update(fullname.encode("utf-8")) - dependency_digest.update(b"\0") - dependency_digest.update(dependency_path.encode("utf-8")) - dependency_digest.update(b"\0") - dependency_digest.update(hashlib.sha256(payload).digest()) - dependency_digest.update(b"\0") - source["dependency_manifest_sha256"] = dependency_digest.hexdigest() - return factory, source class _BindingTimeout(Exception): diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 5f86237..704644a 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -384,7 +384,8 @@ def test_binding_import_runtime_error_is_invalid_evidence(self) -> None: subprocess.run(command, cwd=root, check=True) with patch("sys.path", [str(root), *sys.path]): with self.assertRaisesRegex(ConformanceInputError, "RuntimeError"): - load_binding_factory("broken:factory", root) + with load_binding_factory("broken:factory", root): + pass def test_binding_import_has_a_hard_timeout(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: @@ -405,7 +406,8 @@ def test_binding_import_has_a_hard_timeout(self) -> None: "prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01 ): with self.assertRaisesRegex(ConformanceInputError, "import timed out"): - load_binding_factory("slow:factory", root) + with load_binding_factory("slow:factory", root): + pass def test_binding_executes_the_admitted_bytes_not_a_reopened_path(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: @@ -434,7 +436,8 @@ def substitute_after_read(path: Path) -> bytes: with patch("pathlib.Path.read_bytes", substitute_after_read): with self.assertRaisesRegex(ConformanceInputError, "dirty"): - load_binding_factory("stable:factory", root) + with load_binding_factory("stable:factory", root) as (factory, _): + self.assertEqual(factory(), "ADMITTED") def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: @@ -463,16 +466,16 @@ def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> No prior = sys.modules.get("bindingpkg.helper") sys.modules["bindingpkg.helper"] = cached try: - factory, identity = load_binding_factory("bindingpkg:factory", root) + with load_binding_factory("bindingpkg:factory", root) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertRegex( + identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" + ) finally: if prior is None: sys.modules.pop("bindingpkg.helper", None) else: sys.modules["bindingpkg.helper"] = prior - self.assertEqual(factory(), "COMMITTED") - self.assertRegex( - identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" - ) def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: @@ -506,9 +509,12 @@ def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: sys.modules["bindingpkg"] = cached_parent sys.modules["bindingpkg.helper"] = cached_helper try: - factory, identity = load_binding_factory( + with load_binding_factory( "bindingpkg.subbinding:factory", root - ) + ) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertIsNot(sys.modules["bindingpkg"], cached_parent) + self.assertIsNot(sys.modules["bindingpkg.helper"], cached_helper) self.assertIs(sys.modules["bindingpkg"], cached_parent) self.assertIs(sys.modules["bindingpkg.helper"], cached_helper) finally: @@ -520,11 +526,53 @@ def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: sys.modules.pop("bindingpkg.helper", None) else: sys.modules["bindingpkg.helper"] = prior_helper - self.assertEqual(factory(), "COMMITTED") self.assertRegex( identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" ) + def test_lazy_dependency_import_remains_isolated_for_factory_lifetime(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "subbinding.py").write_text( + "def factory():\n" + " from .helper import VALUE\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached_parent = types.ModuleType("bindingpkg") + cached_parent.__path__ = [str(package)] + cached_helper = types.ModuleType("bindingpkg.helper") + cached_helper.VALUE = "CACHED-UNBOUND" + saved = {name: sys.modules.get(name) for name in ("bindingpkg", "bindingpkg.helper")} + sys.modules["bindingpkg"] = cached_parent + sys.modules["bindingpkg.helper"] = cached_helper + try: + with load_binding_factory( + "bindingpkg.subbinding:factory", root + ) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertRegex(identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$") + finally: + for name, module in saved.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory)