From d16089a0c7b93d359980f142982205b671360dfa Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 15:56:25 +0200 Subject: [PATCH 01/88] docs: define LLM Errata validation program --- INDEPENDENT_IMPLEMENTATION.md | 39 ++++++ REVIEW_REQUEST.md | 47 ++++++++ docs/PUBLICATION_STRATEGY.md | 72 +++++++++++ ...6-08-12-semantic-probes-and-publication.md | 95 +++++++++++++++ .../2026-08-12-semantic-probes-design.md | 92 ++++++++++++++ ...2-validation-publication-program-design.md | 113 ++++++++++++++++++ 6 files changed, 458 insertions(+) create mode 100644 INDEPENDENT_IMPLEMENTATION.md create mode 100644 REVIEW_REQUEST.md create mode 100644 docs/PUBLICATION_STRATEGY.md create mode 100644 docs/superpowers/plans/2026-08-12-semantic-probes-and-publication.md create mode 100644 docs/superpowers/specs/2026-08-12-semantic-probes-design.md create mode 100644 docs/superpowers/specs/2026-08-12-validation-publication-program-design.md diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md new file mode 100644 index 0000000..f2232b0 --- /dev/null +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -0,0 +1,39 @@ +# Call for independent implementations + +LLM Errata needs independently authored adapters and a separately produced +validator before it can claim interoperability. + +Repository: https://github.com/thomaswillner/llm-errata + +Current verdict: **NOT_PROD_READY**. + +## Requested work + +Implement one adapter from the published schemas, vectors, and conformance +requirements without copying implementation code from `prototype/`. + +Candidate substrates include: + +- file-backed Markdown or knowledge-base memory; +- vector memory; +- graph memory; +- an agent framework's durable memory store; and +- an intentionally opaque or incomplete importer. + +The implementation must preserve correction, supersession, and erasure as +different operations; quarantine affected state before repair; run negative, +positive, and preservation checks; and report `partial`, `unknown`, or `failed` +without converting missing evidence into success. + +## Independence and evidence + +An implementation report must name its authors, repository and commit, supported +specification version, dependencies, test commands, unsupported behavior, and +licence. Shared conformance vectors are expected. Shared reference-adapter code +disqualifies the implementation as independent evidence. + +The current personal-use licence requires written permission to implement the +specification. Open a GitHub issue naming the research or standards scope, +repository, organization, requested version, and intended publication. A +permission grant does not imply technical endorsement. + diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md new file mode 100644 index 0000000..8815fb5 --- /dev/null +++ b/REVIEW_REQUEST.md @@ -0,0 +1,47 @@ +# Independent review request + +LLM Errata is seeking independent technical review of an experimental +conformance proposal for repairing derived AI memory after a post-export +correction, supersession, or erasure. + +Repository: https://github.com/thomaswillner/llm-errata + +Current verdict: **NOT_PROD_READY**. + +## Reviews requested + +### Novelty and conformance review + +Challenge the narrow conjunction in `RESEARCH.md` and `PRIOR_ART.md`. Identify +any earlier public implementation or normative profile that requires all of: + +1. post-export update delivery to prior importers; +2. importer-side quarantine and repair of known local descendants; +3. negative, positive, and preservation verification; and +4. a signed, coverage-aware callback bound to the event and importer state. + +Partial collisions are valuable and should narrow the proposal. + +### Security and distributed-systems review + +Review authorization, sequencing, rollback, equivocation, quarantine ordering, +lineage gaps, opaque coverage, receipt binding, erasure confidentiality, key +lifecycle, retries, and incomplete stores. Find the smallest counterexample that +breaks an invariant or makes the contract impractical. + +## Required review record + +Please identify: + +- your name or stable public identity; +- reviewed commit SHA and date; +- files and behavior reviewed; +- prior work or standards relied upon; +- blocking, major, and minor findings; +- conflicts of interest or prior involvement; and +- verdict: proceed, narrow, redesign, or retire. + +Open a GitHub issue for public findings. Security-sensitive findings must follow +`SECURITY.md`. Review invitations and automated reviews are not independent +evidence for the production-readiness ledger. + diff --git a/docs/PUBLICATION_STRATEGY.md b/docs/PUBLICATION_STRATEGY.md new file mode 100644 index 0000000..c7a77d9 --- /dev/null +++ b/docs/PUBLICATION_STRATEGY.md @@ -0,0 +1,72 @@ +# Publication strategy + +## Canonical destination + +Every publication points to: + +https://github.com/thomaswillner/llm-errata + +Substantive prior-art, conformance, implementation, or security findings return +to traceable GitHub issues, pull requests, or private security reports. + +## Message + +**Problem:** portable AI memory can be corrected at its source while stale local +summaries, embeddings, profiles, graphs, and caches remain active elsewhere. + +**Proposal:** importers observe authenticated errata, quarantine known +descendants, rebuild from valid inputs, test absence/replacement/preservation, +and attest honestly—including `unknown` coverage. + +**Bounded claim:** reviewed sources contain the individual mechanisms, but no +reviewed public implementation or normative profile required the complete +conjunction at the dated cutoff. + +**Status:** experimental conformance proposal and tested reference +implementation; `NOT_PROD_READY`; no independent interoperability evidence yet. + +**Call:** find prior art, break the invariants, review the conformance surface, +or independently implement an adapter. + +## Channel matrix + +| Channel | Audience | Primary request | Publication gate | +|---|---|---|---| +| GitHub | implementers and reviewers | issues, review, adapters | repository and CI current | +| Hacker News | systems and open-source engineers | falsification and prior art | concise factual submission | +| LinkedIn | named researchers and standards experts | independent reviewers | authenticated author account | +| X | AI-memory practitioners | reviewer and implementer referrals | authenticated account | +| Reddit | focused technical communities | critique, not promotion | subreddit rules checked | +| DEV Community | developers | reproducible technical walkthrough | authenticated account | +| Medium | broader technical readers | explanatory article | authenticated account | +| Zenodo/OSF | citable archival record | archive release and evidence | stable reviewed release | +| arXiv | academic readers | paper and experiments | paper-quality manuscript | + +## Canonical short announcement + +> If an AI imports a memory, should it also accept responsibility for future +> corrections? LLM Errata is an experimental vendor-neutral conformance proposal: +> observe an authenticated correction, quarantine known descendants, rebuild +> from valid inputs, run negative/positive/preservation checks, and return a +> signed receipt that admits unknown coverage. The repository includes prior-art +> collisions, schemas, vectors, a runnable reference implementation, and explicit +> falsifiers. It is NOT_PROD_READY and needs independent review and adapters: +> https://github.com/thomaswillner/llm-errata + +## Hacker News title + +`LLM Errata – making corrections follow portable AI memory` + +## LinkedIn opening + +`An imported AI memory is a dependency, not a copy.` + +The post should ask AI-memory, distributed-systems, privacy, and standards +specialists to review the exact four-part conjunction and identify collisions. + +## Publication evidence + +For every successful post, record channel, URL, publication timestamp, account, +source commit, exact copy digest, and moderation state. A submitted or queued +post is not recorded as publicly available until its URL is accessible. + diff --git a/docs/superpowers/plans/2026-08-12-semantic-probes-and-publication.md b/docs/superpowers/plans/2026-08-12-semantic-probes-and-publication.md new file mode 100644 index 0000000..fe911ec --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-semantic-probes-and-publication.md @@ -0,0 +1,95 @@ +# Semantic Probes and Publication Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Complete the internal Phase 2 semantic-probe surface, publish independent-review and implementation calls, and expose the work through an evidence-bounded draft PR. + +**Architecture:** A provider-neutral semantic module records exact verifier configuration and validates structured observations. A recorded-fixture adapter makes conformance deterministic and offline. Publication artifacts remain documentation and GitHub calls; they never change readiness gates without external evidence. + +**Tech Stack:** Python 3.11+ standard library, `dataclasses`, `enum`, `hashlib`, `json`, `argparse`, `unittest`, GitHub Actions. + +## Global Constraints + +- Preserve correction, supersession, and erasure as distinct operations. +- Preserve quarantine-before-repair and negative-positive-preservation semantics. +- Never convert missing, inconclusive, malformed, configuration-drifted, or provider-error evidence into success. +- Erasure reports and fixtures must not retain the erased value. +- `make check` remains offline, standard-library-only, and green. +- Internal implementation cannot change G2 from `BLOCKED`; dated independent review is also required. +- Publication must state `NOT_PROD_READY` and link the canonical repository. + +--- + +### Task 1: Provider-neutral semantic evidence model + +**Files:** +- Create: `prototype/semantic.py` +- Test: `tests/test_semantic.py` + +**Interfaces:** +- Produces: `ProbeKind`, `ObservationVerdict`, `SemanticCoverage`, `SemanticProbe`, `VerifierConfig`, `SemanticObservation`, `SemanticProbeReport`, `SemanticVerifier`, `RecordedSemanticVerifier`, and `SemanticProbeRunner`. + +- [ ] Write tests for canonical configuration digests, all-pass verification, required failure, inconclusive/error/missing/duplicate/configuration-drift results, deterministic serialization, and erasure non-disclosure. +- [ ] Run `python3 -m unittest tests.test_semantic -v` and confirm the new module is missing. +- [ ] Implement immutable data models, canonical JSON serialization, strict parsing, the verifier protocol, recorded adapter, and fail-closed aggregation. +- [ ] Run the focused tests and confirm they pass. +- [ ] Commit with `feat: add provider-neutral semantic probe evidence`. + +### Task 2: Offline conformance fixtures and CLI + +**Files:** +- Create: `spec/semantic/probes.json` +- Create: `spec/semantic/verifier-config.json` +- Create: `spec/semantic/observations.json` +- Modify: `prototype/cli.py` +- Modify: `spec/README.md` +- Modify: `prototype/README.md` +- Test: `tests/test_cli.py` +- Test: `tests/test_semantic.py` + +**Interfaces:** +- Consumes: strict `from_dict` methods and `SemanticProbeRunner.run` from Task 1. +- Produces: `errata semantic-test --probes PATH --config PATH --observations PATH` with exit `0`, `1`, or `2`. + +- [ ] Add failing CLI tests for verified, failed, and unknown fixture sets plus malformed input. +- [ ] Add synthetic manifests containing no real or erased personal data. +- [ ] Implement the CLI command and canonical JSON report output. +- [ ] Document fixture format, command, exit codes, privacy boundary, and provider-adapter seam. +- [ ] Run focused CLI and semantic tests. +- [ ] Commit with `feat: publish semantic probe conformance fixtures`. + +### Task 3: Readiness, roadmap, and publication integration + +**Files:** +- Modify: `ROADMAP.md` +- Modify: `README.md` +- Modify: `PRODUCTION_READINESS.md` +- Modify: `readiness/production-readiness.json` +- Modify: `CHANGELOG.md` +- Modify: `PUBLISHING.md` +- Modify: `scripts/validate_repo.py` +- Modify: `tests/test_validate_repo.py` + +**Interfaces:** +- Consumes: Task 1 and Task 2 artifacts. +- Produces: accurate Phase 2 item 6 implementation status while G2 remains `BLOCKED` pending independent review. + +- [ ] Add regression tests requiring review and implementation-call artifacts and preventing Phase 2 completion from upgrading G2. +- [ ] Update maturity language, roadmap status, evidence references, and publication workflow. +- [ ] Run `make check` and `git diff --check`. +- [ ] Commit with `docs: open independent LLM Errata validation program`. + +### Task 4: Review, push, and public GitHub calls + +**Files:** +- Review all changes from the branch base. + +**Interfaces:** +- Produces: pushed branch, draft PR, reviewer recruitment issue, independent-implementation issue, and public URL ledger. + +- [ ] Conduct separate specification and code-quality review; repair every load-bearing finding. +- [ ] Run `make check`, `make links`, `git diff --check`, and secret-safe staged-content inspection. +- [ ] Push `agent/g2-publication` and create a draft PR targeting `agent/prod-readiness` until PR #3 merges. +- [ ] Create GitHub issues for independent review, implementation permission/recruitment, and Phase 3 system nominations. +- [ ] Publish authenticated external announcements where platform access and rules permit, then record exact URLs without upgrading readiness gates. + diff --git a/docs/superpowers/specs/2026-08-12-semantic-probes-design.md b/docs/superpowers/specs/2026-08-12-semantic-probes-design.md new file mode 100644 index 0000000..e155149 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-semantic-probes-design.md @@ -0,0 +1,92 @@ +# Provider-Neutral Semantic Probes Design + +**Date:** 2026-08-12 + +**Status:** Approved as Phase 2 item 6 implementation scope + +## Purpose + +Add model-assisted behavioral evidence without coupling LLM Errata to one +provider or allowing nondeterministic, unavailable, or ambiguous verification +to become success. Structural checks remain deterministic and separate. + +## Architecture + +`prototype/semantic.py` defines four stable units: + +1. `SemanticProbe` describes one negative, positive, or preservation question, + its declared scope, operation, and whether it is required. +2. `VerifierConfig` records provider, model, prompt-template version, sampling + parameters, and an optional seed. Its canonical digest binds observations to + the exact verifier configuration. +3. `SemanticVerifier` is a protocol whose only operation evaluates one probe and + returns a structured observation. Provider SDKs remain outside the core. +4. `SemanticProbeRunner` validates observations and produces a coverage-aware + report. A deterministic `RecordedSemanticVerifier` consumes checked-in + synthetic fixtures for tests and offline conformance. + +The runner never interprets free-form prose as success. A verifier observation +must declare `pass`, `fail`, `inconclusive`, or `error`, bind the probe ID and +configuration digest, carry a timestamp, and include a SHA-256 digest of its raw +response. Raw provider output is not included in receipts. + +## Aggregation + +- all required observations valid and `pass` -> `verified`; +- any valid required `fail` -> `failed`; +- otherwise -> `unknown`. + +`inconclusive`, `error`, missing observations, duplicates, unexpected probe IDs, +configuration drift, malformed timestamps, or invalid digests all yield +`unknown` with explicit limitations. Optional probes never compensate for a +missing required probe. + +## Erasure boundary + +Erasure evidence must not reproduce erased content. An erasure probe record may +contain opaque commitments, synthetic labels, and a content-free prompt +template, but not the retired value. Runtime provider adapters may receive +ephemeral confidential inputs through provider-owned mechanisms; those inputs +are excluded from the persisted probe, observation, report, and receipt. + +The deterministic fixtures use synthetic propositions and still assert that the +serialized report contains no retired erasure value. + +## CLI and fixtures + +`errata semantic-test` accepts a probe manifest, verifier configuration, and +recorded-observation manifest. It prints the canonical report and exits: + +- `0` for `verified`; +- `1` for `failed`; +- `2` for `unknown`. + +Checked-in fixtures cover: + +- correction with all required probes passing; +- supersession with a failing negative probe; +- erasure with content-free records; +- inconclusive observation; +- provider error; +- missing response; +- duplicate response; +- configuration drift; and +- deliberately nonconforming verifier output. + +## Integration boundary + +This milestone publishes semantic evidence as a separate report and CLI command. +It does not silently change existing Phase 1 receipt semantics. A later reviewed +profile may embed or reference the semantic report from a receipt after external +conformance review settles binding and privacy questions. + +## Tests and evidence + +Unit tests must prove aggregation, binding, parser failure behavior, erasure +non-disclosure, deterministic serialization, and CLI exit codes. `make check` +must continue to run without network access or API keys. + +Implementation completes Phase 2 item 6 internally. G2 remains `BLOCKED` until +an independent reviewer evaluates the schemas, vectors, CLI, adapters, +cryptographic boundaries, and semantic-probe layer as one conformance surface. + diff --git a/docs/superpowers/specs/2026-08-12-validation-publication-program-design.md b/docs/superpowers/specs/2026-08-12-validation-publication-program-design.md new file mode 100644 index 0000000..f8685a9 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-validation-publication-program-design.md @@ -0,0 +1,113 @@ +# LLM Errata Validation and Publication Program Design + +**Date:** 2026-08-12 + +**Status:** Approved for execution by the repository owner + +## Objective + +Turn LLM Errata from a self-reviewed reference implementation into a publicly +falsifiable interoperability effort. Publication is part of validation: every +artifact must invite prior-art challenges, independent review, and independent +implementation without representing planned work as completed evidence. + +## Fixed claims boundary + +The program may describe LLM Errata as an experimental, vendor-neutral +conformance proposal and tested reference implementation. It may state that no +reviewed public source was found to require the complete four-part conjunction +at the dated research cutoff. + +The program must not describe LLM Errata as: + +- a world first, proven breakthrough, production protocol, or standard; +- proof of deletion, semantic absence, patentability, or freedom to operate; +- interoperable before independently authored implementations and measured + cross-system evidence exist; or +- externally reviewed when the only reviewers are the author or author-directed + agents. + +## Program sequence + +1. **Intellectual-property posture.** Retain the current personal-use licence. + Permit research and standards implementations through explicit written, + scoped grants. Do not relicense or make patent conclusions without separate + owner approval informed by qualified legal advice. +2. **Independent review.** Publish two calls: one for novelty/conformance review + and one for security/distributed-systems review. Reviews must identify the + reviewer, reviewed commit, date, scope, findings, and conflicts. +3. **Readiness-foundation integration.** Preserve PR #3 and branch protection. + It may merge only after the required separate CODEOWNER approval and green + required checks. +4. **G2 semantic evidence.** Implement the provider-neutral semantic-probe + design in the companion specification. Internal implementation can complete + Phase 2 item 6, but G2 remains `BLOCKED` until dated independent review of + the complete Phase 2 surface exists. +5. **Independent conformance and interoperability.** Recruit two independently + authored adapters and a separately produced validator. Only then run the + declared three-system experiment with synthetic data, one intentionally + nonconforming importer, and one genuinely incomplete-coverage system. +6. **Evidence-bounded publication.** Publish channel-specific calls that link + the canonical repository and request falsification, review, or implementation. + Record every public URL and the exact claim version used. + +## Licensing and permission strategy + +The current licence remains controlling. Research implementers may request a +written permission grant through a public GitHub issue. A grant must name: + +- grantee and repository or organization; +- research or standards purpose; +- permitted specification version and files; +- commercial and redistribution boundaries; +- term, attribution, and publication expectations; and +- whether resulting implementation evidence may be cited by LLM Errata. + +No permission request, discussion, or contribution changes the licence by +implication. Permission records must not disclose confidential legal advice. + +## Independent evidence contract + +An independent review qualifies only when its producer is not the author, not +the implementation maker being reviewed, and not an agent controlled by either. +It must be reproducible from a named commit and publish or privately deliver a +dated report with concrete findings. A review invitation is not review evidence. + +An independently authored adapter qualifies only when its author works from the +published specification and vectors rather than copying the reference adapter. +Shared test vectors are allowed and expected. Shared implementation code is not. + +## Publication channels + +Priority is based on evidence value rather than audience size: + +1. GitHub repository, issues, pull requests, and Discussions if enabled; +2. Hacker News and relevant technical forums for prior-art and systems critique; +3. LinkedIn and X for named expert recruitment; +4. Reddit communities whose rules permit research or open-source submissions; +5. archival research surfaces such as Zenodo or OSF after a citable release; +6. arXiv only after a paper-quality manuscript and appropriate category fit. + +Every post links `https://github.com/thomaswillner/llm-errata`, uses the current +`NOT_PROD_READY` disclosure, and directs substantive findings into traceable +GitHub issues or pull requests. + +## Acceptance gates + +- Strategy, reviewer rubric, implementation call, and publication copy are + version-controlled and pass repository checks. +- G2 deterministic fixtures pass without credentials or network access. +- Missing, skipped, malformed, configuration-drifted, or inconclusive semantic + observations cannot support a verified semantic result. +- Recruitment and publication URLs are recorded after successful submission. +- Readiness gates change only from qualifying evidence, never from invitations, + implementation completion, CI, or publication volume. + +## Stop and reroute conditions + +Stop mutation or publication when copy would overstate evidence, platform rules +prohibit the submission, credentials are unavailable, a reviewer is not +independent, or implementation requires an unapproved licence change. Reroute +after a complete prior-art collision, external review rejection, material G2 +contract change, or failure to recruit independent implementers. + From bcee821e2aad35f2a1666257fbc46608336da2b8 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 16:04:46 +0200 Subject: [PATCH 02/88] feat: add provider-neutral semantic probe evidence --- prototype/semantic.py | 435 +++++++++++++++++++++++++++++++++++++++++ tests/test_semantic.py | 216 ++++++++++++++++++++ 2 files changed, 651 insertions(+) create mode 100644 prototype/semantic.py create mode 100644 tests/test_semantic.py diff --git a/prototype/semantic.py b/prototype/semantic.py new file mode 100644 index 0000000..b231c75 --- /dev/null +++ b/prototype/semantic.py @@ -0,0 +1,435 @@ +"""Provider-neutral semantic probe evidence. + +This module records scoped behavioural evidence. It deliberately does not +interpret provider prose: an adapter must return a structured verdict bound to +one declared probe and one exact verifier configuration. Structural repair +checks remain the controller's separate responsibility. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from types import MappingProxyType +from typing import Any, Iterable, Mapping, Protocol, Sequence + + +def _canonical_json(value: object) -> str: + try: + return json.dumps(value, allow_nan=False, separators=(",", ":"), sort_keys=True) + except (TypeError, ValueError) as error: + raise ValueError("value must be JSON-serializable") from error + + +def _mapping(value: object, *, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping) or not all(isinstance(key, str) for key in value): + raise ValueError(f"{name} must be an object with string keys") + return value + + +def _exact_fields(value: object, *, fields: frozenset[str], name: str) -> Mapping[str, Any]: + payload = _mapping(value, name=name) + received = frozenset(payload) + if received != fields: + missing = sorted(fields - received) + unexpected = sorted(received - fields) + raise ValueError(f"{name} fields mismatch: missing={missing}, unexpected={unexpected}") + return payload + + +def _nonempty_string(value: object, *, name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{name} must be a non-empty string") + return value + + +def _sha256(value: object, *, name: str) -> str: + text = _nonempty_string(value, name=name) + if len(text) != 64 or any(char not in "0123456789abcdef" for char in text): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return text + + +def _timestamp(value: object, *, name: str) -> str: + text = _nonempty_string(value, name=name) + if not text.endswith("Z"): + raise ValueError(f"{name} must be an RFC 3339 UTC timestamp") + try: + datetime.fromisoformat(text[:-1] + "+00:00") + except ValueError as error: + raise ValueError(f"{name} must be an RFC 3339 UTC timestamp") from error + return text + + +class ProbeKind(str, Enum): + """The repair-triad question represented by a semantic probe.""" + + NEGATIVE = "negative" + POSITIVE = "positive" + PRESERVATION = "preservation" + + +class ObservationVerdict(str, Enum): + """A verifier's explicit result. Prose cannot substitute for one.""" + + PASS = "pass" + FAIL = "fail" + INCONCLUSIVE = "inconclusive" + ERROR = "error" + + +class SemanticCoverage(str, Enum): + """Terminal coverage of declared required semantic probes.""" + + VERIFIED = "verified" + UNKNOWN = "unknown" + FAILED = "failed" + + +@dataclass(frozen=True) +class SemanticProbe: + """One declared, provider-neutral question over a bounded scope. + + Erasure records intentionally have no retired-value field. Provider + adapters may receive confidential material ephemerally, but persisted + probes, observations, and reports cannot carry it through this interface. + """ + + probe_id: str + kind: ProbeKind + operation: str + scope: str + prompt_template: str + required: bool = True + + def __post_init__(self) -> None: + _nonempty_string(self.probe_id, name="probe_id") + if not isinstance(self.kind, ProbeKind): + raise ValueError("kind must be a ProbeKind") + _nonempty_string(self.operation, name="operation") + _nonempty_string(self.scope, name="scope") + _nonempty_string(self.prompt_template, name="prompt_template") + if not isinstance(self.required, bool): + raise ValueError("required must be boolean") + if self.operation == "erase" and self.kind is ProbeKind.POSITIVE: + raise ValueError("erasure has no positive replacement probe") + + def to_dict(self) -> dict[str, Any]: + return { + "probe_id": self.probe_id, + "kind": self.kind.value, + "operation": self.operation, + "scope": self.scope, + "prompt_template": self.prompt_template, + "required": self.required, + } + + @classmethod + def from_dict(cls, value: object) -> "SemanticProbe": + payload = _exact_fields( + value, + fields=frozenset( + {"probe_id", "kind", "operation", "scope", "prompt_template", "required"} + ), + name="semantic probe", + ) + try: + kind = ProbeKind(payload["kind"]) + except (TypeError, ValueError) as error: + raise ValueError("semantic probe kind is invalid") from error + return cls( + probe_id=_nonempty_string(payload["probe_id"], name="probe_id"), + kind=kind, + operation=_nonempty_string(payload["operation"], name="operation"), + scope=_nonempty_string(payload["scope"], name="scope"), + prompt_template=_nonempty_string( + payload["prompt_template"], name="prompt_template" + ), + required=payload["required"], + ) + + +@dataclass(frozen=True) +class VerifierConfig: + """Exact configuration that binds observations to one verifier setup.""" + + provider: str + model: str + prompt_template_version: str + sampling: Mapping[str, Any] + seed: int | None = None + + def __post_init__(self) -> None: + _nonempty_string(self.provider, name="provider") + _nonempty_string(self.model, name="model") + _nonempty_string(self.prompt_template_version, name="prompt_template_version") + sampling = _mapping(self.sampling, name="sampling") + # Serializing here rejects NaN, functions, and other non-portable values + # before a digest can be advertised as a stable binding. + _canonical_json(dict(sampling)) + object.__setattr__(self, "sampling", MappingProxyType(dict(sampling))) + if self.seed is not None and (not isinstance(self.seed, int) or isinstance(self.seed, bool)): + raise ValueError("seed must be an integer or null") + + def to_dict(self) -> dict[str, Any]: + return { + "provider": self.provider, + "model": self.model, + "prompt_template_version": self.prompt_template_version, + "sampling": dict(self.sampling), + "seed": self.seed, + } + + @property + def digest(self) -> str: + return hashlib.sha256(_canonical_json(self.to_dict()).encode("utf-8")).hexdigest() + + @classmethod + def from_dict(cls, value: object) -> "VerifierConfig": + payload = _exact_fields( + value, + fields=frozenset( + {"provider", "model", "prompt_template_version", "sampling", "seed"} + ), + name="verifier config", + ) + return cls( + provider=_nonempty_string(payload["provider"], name="provider"), + model=_nonempty_string(payload["model"], name="model"), + prompt_template_version=_nonempty_string( + payload["prompt_template_version"], name="prompt_template_version" + ), + sampling=_mapping(payload["sampling"], name="sampling"), + seed=payload["seed"], + ) + + +@dataclass(frozen=True) +class SemanticObservation: + """Persistable verifier result, excluding raw provider output.""" + + probe_id: str + verdict: ObservationVerdict + config_digest: str + observed_at: str + response_digest: str + + def __post_init__(self) -> None: + _nonempty_string(self.probe_id, name="probe_id") + if not isinstance(self.verdict, ObservationVerdict): + raise ValueError("verdict must be an ObservationVerdict") + _sha256(self.config_digest, name="config_digest") + _timestamp(self.observed_at, name="observed_at") + _sha256(self.response_digest, name="response_digest") + + def to_dict(self) -> dict[str, str]: + return { + "probe_id": self.probe_id, + "verdict": self.verdict.value, + "config_digest": self.config_digest, + "observed_at": self.observed_at, + "response_digest": self.response_digest, + } + + @classmethod + def from_dict(cls, value: object) -> "SemanticObservation": + payload = _exact_fields( + value, + fields=frozenset( + {"probe_id", "verdict", "config_digest", "observed_at", "response_digest"} + ), + name="semantic observation", + ) + try: + verdict = ObservationVerdict(payload["verdict"]) + except (TypeError, ValueError) as error: + raise ValueError("semantic observation verdict is invalid") from error + return cls( + probe_id=_nonempty_string(payload["probe_id"], name="probe_id"), + verdict=verdict, + config_digest=_sha256(payload["config_digest"], name="config_digest"), + observed_at=_timestamp(payload["observed_at"], name="observed_at"), + response_digest=_sha256(payload["response_digest"], name="response_digest"), + ) + + +class SemanticVerifier(Protocol): + """Provider adapter seam. SDKs and confidential runtime inputs stay out.""" + + def evaluate( + self, probe: SemanticProbe, config: VerifierConfig + ) -> SemanticObservation | None: ... + + +class RecordedSemanticVerifier: + """Offline verifier backed by checked-in structured observations.""" + + def __init__(self, observations: Iterable[SemanticObservation]) -> None: + self._observations = tuple(observations) + if not all(isinstance(item, SemanticObservation) for item in self._observations): + raise ValueError("recorded observations must be SemanticObservation instances") + + @property + def observations(self) -> tuple[SemanticObservation, ...]: + return self._observations + + def evaluate( + self, probe: SemanticProbe, config: VerifierConfig + ) -> SemanticObservation | None: + matches = [item for item in self._observations if item.probe_id == probe.probe_id] + return matches[0] if len(matches) == 1 else None + + +@dataclass(frozen=True) +class SemanticProbeReport: + """Canonical, coverage-aware semantic evidence report.""" + + config_digest: str + coverage: SemanticCoverage + probes: tuple[SemanticProbe, ...] + observations: tuple[SemanticObservation, ...] + limitations: tuple[str, ...] + + def __post_init__(self) -> None: + _sha256(self.config_digest, name="config_digest") + if not isinstance(self.coverage, SemanticCoverage): + raise ValueError("coverage must be a SemanticCoverage") + if not all(isinstance(item, SemanticProbe) for item in self.probes): + raise ValueError("probes must be SemanticProbe instances") + if not all(isinstance(item, SemanticObservation) for item in self.observations): + raise ValueError("observations must be SemanticObservation instances") + if not all(isinstance(item, str) and item for item in self.limitations): + raise ValueError("limitations must contain non-empty strings") + + def to_dict(self) -> dict[str, Any]: + return { + "config_digest": self.config_digest, + "coverage": self.coverage.value, + "probes": [item.to_dict() for item in sorted(self.probes, key=lambda item: item.probe_id)], + "observations": [ + item.to_dict() + for item in sorted( + self.observations, + key=lambda item: (item.probe_id, item.observed_at, item.response_digest), + ) + ], + "limitations": list(sorted(self.limitations)), + } + + def canonical_json(self) -> str: + return _canonical_json(self.to_dict()) + + @classmethod + def from_dict(cls, value: object) -> "SemanticProbeReport": + payload = _exact_fields( + value, + fields=frozenset( + {"config_digest", "coverage", "probes", "observations", "limitations"} + ), + name="semantic probe report", + ) + if not isinstance(payload["probes"], list) or not isinstance(payload["observations"], list): + raise ValueError("semantic probe report probes and observations must be arrays") + if not isinstance(payload["limitations"], list): + raise ValueError("semantic probe report limitations must be an array") + try: + coverage = SemanticCoverage(payload["coverage"]) + except (TypeError, ValueError) as error: + raise ValueError("semantic probe report coverage is invalid") from error + return cls( + config_digest=_sha256(payload["config_digest"], name="config_digest"), + coverage=coverage, + probes=tuple(SemanticProbe.from_dict(item) for item in payload["probes"]), + observations=tuple( + SemanticObservation.from_dict(item) for item in payload["observations"] + ), + limitations=tuple( + _nonempty_string(item, name="limitation") for item in payload["limitations"] + ), + ) + + +class SemanticProbeRunner: + """Validate probe evidence and aggregate it without upgrading uncertainty.""" + + def run( + self, + probes: Sequence[SemanticProbe], + config: VerifierConfig, + verifier: SemanticVerifier | Iterable[SemanticObservation], + ) -> SemanticProbeReport: + declared = tuple(probes) + if not all(isinstance(item, SemanticProbe) for item in declared): + raise ValueError("probes must be SemanticProbe instances") + ids = [item.probe_id for item in declared] + if len(ids) != len(set(ids)): + raise ValueError("probe IDs must be unique") + + if isinstance(verifier, RecordedSemanticVerifier): + received = verifier.observations + elif hasattr(verifier, "evaluate"): + received = tuple(verifier.evaluate(item, config) for item in declared) # type: ignore[union-attr] + received = tuple(item for item in received if item is not None) + else: + received = tuple(verifier) + if not all(isinstance(item, SemanticObservation) for item in received): + raise ValueError("observations must be SemanticObservation instances") + + limitations: list[str] = [] + declared_by_id = {item.probe_id: item for item in declared} + received_by_id: dict[str, list[SemanticObservation]] = {} + for item in received: + received_by_id.setdefault(item.probe_id, []).append(item) + + for probe_id in sorted(received_by_id): + records = received_by_id[probe_id] + if probe_id not in declared_by_id: + limitations.append(f"unexpected observation for probe {probe_id}") + elif len(records) != 1: + limitations.append(f"duplicate observations for probe {probe_id}") + elif records[0].config_digest != config.digest: + limitations.append(f"configuration drift for probe {probe_id}") + + valid: dict[str, SemanticObservation] = {} + for probe in declared: + records = received_by_id.get(probe.probe_id, []) + if not records: + if probe.required: + limitations.append(f"missing required observation for probe {probe.probe_id}") + continue + if len(records) != 1 or records[0].config_digest != config.digest: + continue + record = records[0] + if probe.required and record.verdict in { + ObservationVerdict.INCONCLUSIVE, + ObservationVerdict.ERROR, + }: + limitations.append( + f"required probe {probe.probe_id} returned {record.verdict.value}" + ) + valid[probe.probe_id] = record + + required = tuple(probe for probe in declared if probe.required) + if not required: + limitations.append("no required semantic probes were declared") + + if limitations: + coverage = SemanticCoverage.UNKNOWN + elif any( + valid[probe.probe_id].verdict is ObservationVerdict.FAIL + for probe in required + ): + coverage = SemanticCoverage.FAILED + else: + coverage = SemanticCoverage.VERIFIED + + return SemanticProbeReport( + config_digest=config.digest, + coverage=coverage, + probes=declared, + observations=tuple(received), + limitations=tuple(sorted(set(limitations))), + ) diff --git a/tests/test_semantic.py b/tests/test_semantic.py new file mode 100644 index 0000000..b340206 --- /dev/null +++ b/tests/test_semantic.py @@ -0,0 +1,216 @@ +"""Provider-neutral, fail-closed semantic probe evidence.""" + +from __future__ import annotations + +import hashlib +import json +import unittest + +from prototype.semantic import ( + ObservationVerdict, + ProbeKind, + RecordedSemanticVerifier, + SemanticCoverage, + SemanticObservation, + SemanticProbe, + SemanticProbeRunner, + VerifierConfig, +) + + +CONFIG = VerifierConfig( + provider="synthetic-provider", + model="synthetic-model-1", + prompt_template_version="2026-08-12", + sampling={"temperature": 0}, + seed=7, +) + + +def probe( + probe_id: str, kind: ProbeKind, *, required: bool = True, operation: str = "correct" +) -> SemanticProbe: + return SemanticProbe( + probe_id=probe_id, + kind=kind, + operation=operation, + scope="current-answer-sample", + prompt_template=f"synthetic-{kind.value}-prompt", + required=required, + ) + + +def observation( + probe_id: str, verdict: ObservationVerdict, *, config: VerifierConfig = CONFIG +) -> SemanticObservation: + return SemanticObservation( + probe_id=probe_id, + verdict=verdict, + config_digest=config.digest, + observed_at="2026-08-12T09:00:00Z", + response_digest=hashlib.sha256(probe_id.encode("utf-8")).hexdigest(), + ) + + +class VerifierConfiguration(unittest.TestCase): + def test_configuration_digest_is_canonical_across_mapping_order(self) -> None: + reordered = VerifierConfig( + provider="synthetic-provider", + model="synthetic-model-1", + prompt_template_version="2026-08-12", + sampling={"temperature": 0, "top_p": 1}, + seed=7, + ) + ordered = VerifierConfig( + provider="synthetic-provider", + model="synthetic-model-1", + prompt_template_version="2026-08-12", + sampling={"top_p": 1, "temperature": 0}, + seed=7, + ) + self.assertEqual(reordered.digest, ordered.digest) + self.assertEqual(len(reordered.digest), 64) + + +class Aggregation(unittest.TestCase): + def setUp(self) -> None: + self.probes = ( + probe("negative", ProbeKind.NEGATIVE), + probe("positive", ProbeKind.POSITIVE), + probe("preserve", ProbeKind.PRESERVATION), + ) + + def report_for(self, observations: tuple[SemanticObservation, ...]): + return SemanticProbeRunner().run( + self.probes, CONFIG, RecordedSemanticVerifier(observations) + ) + + def test_all_required_passing_observations_are_verified(self) -> None: + report = self.report_for( + tuple(observation(item.probe_id, ObservationVerdict.PASS) for item in self.probes) + ) + self.assertEqual(report.coverage, SemanticCoverage.VERIFIED) + self.assertEqual(report.limitations, ()) + + def test_required_failure_is_failed(self) -> None: + report = self.report_for( + ( + observation("negative", ObservationVerdict.FAIL), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + ) + ) + self.assertEqual(report.coverage, SemanticCoverage.FAILED) + + def test_inconclusive_or_error_required_observations_are_unknown(self) -> None: + for verdict in (ObservationVerdict.INCONCLUSIVE, ObservationVerdict.ERROR): + with self.subTest(verdict=verdict): + report = self.report_for( + ( + observation("negative", verdict), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + ) + ) + self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) + self.assertTrue(report.limitations) + + def test_missing_duplicate_and_configuration_drift_are_unknown(self) -> None: + drifted = VerifierConfig( + provider="other-provider", + model="synthetic-model-1", + prompt_template_version="2026-08-12", + sampling={"temperature": 0}, + ) + cases = { + "missing": ( + observation("negative", ObservationVerdict.PASS), + observation("positive", ObservationVerdict.PASS), + ), + "duplicate": ( + observation("negative", ObservationVerdict.PASS), + observation("negative", ObservationVerdict.PASS), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + ), + "drift": ( + observation("negative", ObservationVerdict.PASS, config=drifted), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + ), + } + for name, records in cases.items(): + with self.subTest(case=name): + report = self.report_for(records) + self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) + self.assertTrue(report.limitations) + + def test_unexpected_or_only_optional_evidence_cannot_verify_coverage(self) -> None: + unexpected = self.report_for( + ( + observation("negative", ObservationVerdict.PASS), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + observation("unrelated", ObservationVerdict.PASS), + ) + ) + optional = SemanticProbeRunner().run( + (probe("optional", ProbeKind.NEGATIVE, required=False),), + CONFIG, + RecordedSemanticVerifier((observation("optional", ObservationVerdict.PASS),)), + ) + self.assertEqual(unexpected.coverage, SemanticCoverage.UNKNOWN) + self.assertEqual(optional.coverage, SemanticCoverage.UNKNOWN) + + +class SerializationAndPrivacy(unittest.TestCase): + def test_report_serialization_is_deterministic(self) -> None: + probes = ( + probe("preserve", ProbeKind.PRESERVATION), + probe("negative", ProbeKind.NEGATIVE), + ) + observations = ( + observation("negative", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + ) + report = SemanticProbeRunner().run(probes, CONFIG, RecordedSemanticVerifier(observations)) + self.assertEqual( + report.canonical_json(), + json.dumps(report.to_dict(), sort_keys=True, separators=(",", ":")), + ) + + def test_erasure_report_does_not_disclose_retired_value(self) -> None: + erased_value = "orchid-lantern-secret" + erasure_probe = probe( + "erase-negative", ProbeKind.NEGATIVE, operation="erase" + ) + report = SemanticProbeRunner().run( + (erasure_probe,), + CONFIG, + RecordedSemanticVerifier( + (observation("erase-negative", ObservationVerdict.PASS),) + ), + ) + self.assertNotIn(erased_value, report.canonical_json()) + + +class StrictParsing(unittest.TestCase): + def test_observation_parser_rejects_free_form_output_and_bad_digest(self) -> None: + payload = observation("negative", ObservationVerdict.PASS).to_dict() + payload["raw_output"] = "pass because retired value was absent" + with self.assertRaises(ValueError): + SemanticObservation.from_dict(payload) + payload.pop("raw_output") + payload["response_digest"] = "not-a-sha256" + with self.assertRaises(ValueError): + SemanticObservation.from_dict(payload) + payload["response_digest"] = observation( + "negative", ObservationVerdict.PASS + ).response_digest + payload["observed_at"] = "not-a-timestamp" + with self.assertRaises(ValueError): + SemanticObservation.from_dict(payload) + + +if __name__ == "__main__": + unittest.main() From adce119e9cdff6dafc5d1a247443f3bb5a843792 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 16:14:30 +0200 Subject: [PATCH 03/88] fix: harden semantic evidence boundaries --- prototype/semantic.py | 138 +++++++++++++++++++++++++++++++++-------- tests/test_semantic.py | 110 ++++++++++++++++++++++++++++---- 2 files changed, 210 insertions(+), 38 deletions(-) diff --git a/prototype/semantic.py b/prototype/semantic.py index b231c75..15eee1a 100644 --- a/prototype/semantic.py +++ b/prototype/semantic.py @@ -64,6 +64,24 @@ def _timestamp(value: object, *, name: str) -> str: return text +def _freeze_json(value: object) -> object: + """Copy and recursively freeze JSON-compatible configuration values.""" + + if isinstance(value, Mapping): + return MappingProxyType({key: _freeze_json(item) for key, item in value.items()}) + if isinstance(value, list): + return tuple(_freeze_json(item) for item in value) + return value + + +def _thaw_json(value: object) -> object: + if isinstance(value, Mapping): + return {key: _thaw_json(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thaw_json(item) for item in value] + return value + + class ProbeKind(str, Enum): """The repair-triad question represented by a semantic probe.""" @@ -89,6 +107,17 @@ class SemanticCoverage(str, Enum): FAILED = "failed" +class SemanticOperation(str, Enum): + """Canonical operation vocabulary retained by semantic evidence.""" + + CORRECTION = "correction" + SUPERSESSION = "supersession" + ERASURE = "erasure" + + +ERASURE_PROMPT_TEMPLATE = "erasure-content-free-v1" + + @dataclass(frozen=True) class SemanticProbe: """One declared, provider-neutral question over a bounded scope. @@ -100,7 +129,7 @@ class SemanticProbe: probe_id: str kind: ProbeKind - operation: str + operation: SemanticOperation scope: str prompt_template: str required: bool = True @@ -109,19 +138,25 @@ def __post_init__(self) -> None: _nonempty_string(self.probe_id, name="probe_id") if not isinstance(self.kind, ProbeKind): raise ValueError("kind must be a ProbeKind") - _nonempty_string(self.operation, name="operation") + if not isinstance(self.operation, SemanticOperation): + raise ValueError("operation must be a SemanticOperation") _nonempty_string(self.scope, name="scope") _nonempty_string(self.prompt_template, name="prompt_template") if not isinstance(self.required, bool): raise ValueError("required must be boolean") - if self.operation == "erase" and self.kind is ProbeKind.POSITIVE: + if self.operation is SemanticOperation.ERASURE and self.kind is ProbeKind.POSITIVE: raise ValueError("erasure has no positive replacement probe") + if ( + self.operation is SemanticOperation.ERASURE + and self.prompt_template != ERASURE_PROMPT_TEMPLATE + ): + raise ValueError("erasure probes must use the fixed content-free prompt template") def to_dict(self) -> dict[str, Any]: return { "probe_id": self.probe_id, "kind": self.kind.value, - "operation": self.operation, + "operation": self.operation.value, "scope": self.scope, "prompt_template": self.prompt_template, "required": self.required, @@ -140,10 +175,14 @@ def from_dict(cls, value: object) -> "SemanticProbe": kind = ProbeKind(payload["kind"]) except (TypeError, ValueError) as error: raise ValueError("semantic probe kind is invalid") from error + try: + operation = SemanticOperation(payload["operation"]) + except (TypeError, ValueError) as error: + raise ValueError("semantic probe operation is invalid") from error return cls( probe_id=_nonempty_string(payload["probe_id"], name="probe_id"), kind=kind, - operation=_nonempty_string(payload["operation"], name="operation"), + operation=operation, scope=_nonempty_string(payload["scope"], name="scope"), prompt_template=_nonempty_string( payload["prompt_template"], name="prompt_template" @@ -169,8 +208,8 @@ def __post_init__(self) -> None: sampling = _mapping(self.sampling, name="sampling") # Serializing here rejects NaN, functions, and other non-portable values # before a digest can be advertised as a stable binding. - _canonical_json(dict(sampling)) - object.__setattr__(self, "sampling", MappingProxyType(dict(sampling))) + normalized = json.loads(_canonical_json(dict(sampling))) + object.__setattr__(self, "sampling", _freeze_json(normalized)) if self.seed is not None and (not isinstance(self.seed, int) or isinstance(self.seed, bool)): raise ValueError("seed must be an integer or null") @@ -179,7 +218,7 @@ def to_dict(self) -> dict[str, Any]: "provider": self.provider, "model": self.model, "prompt_template_version": self.prompt_template_version, - "sampling": dict(self.sampling), + "sampling": _thaw_json(self.sampling), "seed": self.seed, } @@ -339,7 +378,7 @@ def from_dict(cls, value: object) -> "SemanticProbeReport": coverage = SemanticCoverage(payload["coverage"]) except (TypeError, ValueError) as error: raise ValueError("semantic probe report coverage is invalid") from error - return cls( + report = cls( config_digest=_sha256(payload["config_digest"], name="config_digest"), coverage=coverage, probes=tuple(SemanticProbe.from_dict(item) for item in payload["probes"]), @@ -350,6 +389,47 @@ def from_dict(cls, value: object) -> "SemanticProbeReport": _nonempty_string(item, name="limitation") for item in payload["limitations"] ), ) + expected = _coverage_for( + report.probes, report.observations, report.config_digest, report.limitations + ) + if report.coverage is not expected: + raise ValueError("semantic probe report coverage contradicts its evidence") + return report + + +def _required_triad(probes: Sequence[SemanticProbe]) -> tuple[SemanticOperation, set[ProbeKind]]: + operations = {probe.operation for probe in probes} + if len(operations) != 1: + raise ValueError("all probes in a report must have one operation") + operation = next(iter(operations)) + required_kinds = {probe.kind for probe in probes if probe.required} + expected = ( + {ProbeKind.NEGATIVE, ProbeKind.PRESERVATION} + if operation is SemanticOperation.ERASURE + else {ProbeKind.NEGATIVE, ProbeKind.POSITIVE, ProbeKind.PRESERVATION} + ) + if required_kinds != expected: + raise ValueError(f"required probe triad must be exactly {sorted(item.value for item in expected)}") + return operation, expected + + +def _coverage_for( + probes: Sequence[SemanticProbe], + observations: Sequence[SemanticObservation], + config_digest: str, + limitations: Sequence[str], +) -> SemanticCoverage: + required = {probe.probe_id for probe in probes if probe.required} + valid_fails = { + item.probe_id + for item in observations + if item.probe_id in required + and item.config_digest == config_digest + and item.verdict is ObservationVerdict.FAIL + } + if valid_fails: + return SemanticCoverage.FAILED + return SemanticCoverage.UNKNOWN if limitations else SemanticCoverage.VERIFIED class SemanticProbeRunner: @@ -367,18 +447,38 @@ def run( ids = [item.probe_id for item in declared] if len(ids) != len(set(ids)): raise ValueError("probe IDs must be unique") + _required_triad(declared) + adapter_limitations: list[str] = [] if isinstance(verifier, RecordedSemanticVerifier): received = verifier.observations elif hasattr(verifier, "evaluate"): - received = tuple(verifier.evaluate(item, config) for item in declared) # type: ignore[union-attr] - received = tuple(item for item in received if item is not None) + received_items: list[SemanticObservation] = [] + adapter_limitations = [] + for item in declared: + try: + result = verifier.evaluate(item, config) # type: ignore[union-attr] + except Exception as error: + adapter_limitations.append( + f"provider error for probe {item.probe_id}: {type(error).__name__}" + ) + continue + if result is None: + continue + if not isinstance(result, SemanticObservation): + adapter_limitations.append( + f"malformed verifier result for probe {item.probe_id}" + ) + continue + received_items.append(result) + received = tuple(received_items) else: received = tuple(verifier) + adapter_limitations = [] if not all(isinstance(item, SemanticObservation) for item in received): raise ValueError("observations must be SemanticObservation instances") - limitations: list[str] = [] + limitations: list[str] = list(adapter_limitations) declared_by_id = {item.probe_id: item for item in declared} received_by_id: dict[str, list[SemanticObservation]] = {} for item in received: @@ -412,19 +512,7 @@ def run( ) valid[probe.probe_id] = record - required = tuple(probe for probe in declared if probe.required) - if not required: - limitations.append("no required semantic probes were declared") - - if limitations: - coverage = SemanticCoverage.UNKNOWN - elif any( - valid[probe.probe_id].verdict is ObservationVerdict.FAIL - for probe in required - ): - coverage = SemanticCoverage.FAILED - else: - coverage = SemanticCoverage.VERIFIED + coverage = _coverage_for(declared, received, config.digest, limitations) return SemanticProbeReport( config_digest=config.digest, diff --git a/tests/test_semantic.py b/tests/test_semantic.py index b340206..81dc7e2 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -8,11 +8,14 @@ from prototype.semantic import ( ObservationVerdict, + ERASURE_PROMPT_TEMPLATE, ProbeKind, RecordedSemanticVerifier, SemanticCoverage, SemanticObservation, + SemanticOperation, SemanticProbe, + SemanticProbeReport, SemanticProbeRunner, VerifierConfig, ) @@ -28,14 +31,18 @@ def probe( - probe_id: str, kind: ProbeKind, *, required: bool = True, operation: str = "correct" + probe_id: str, kind: ProbeKind, *, required: bool = True, operation: SemanticOperation = SemanticOperation.CORRECTION ) -> SemanticProbe: return SemanticProbe( probe_id=probe_id, kind=kind, operation=operation, scope="current-answer-sample", - prompt_template=f"synthetic-{kind.value}-prompt", + prompt_template=( + ERASURE_PROMPT_TEMPLATE + if operation is SemanticOperation.ERASURE + else f"synthetic-{kind.value}-prompt" + ), required=required, ) @@ -71,6 +78,16 @@ def test_configuration_digest_is_canonical_across_mapping_order(self) -> None: self.assertEqual(reordered.digest, ordered.digest) self.assertEqual(len(reordered.digest), 64) + def test_nested_sampling_is_detached_from_caller_and_serialized_copy(self) -> None: + supplied = {"nested": {"temperature": 0}, "stops": ["END"]} + config = VerifierConfig("p", "m", "v", supplied) + digest = config.digest + supplied["nested"]["temperature"] = 1 + emitted = config.to_dict() + emitted["sampling"]["nested"]["temperature"] = 2 + self.assertEqual(config.digest, digest) + self.assertEqual(config.to_dict()["sampling"]["nested"]["temperature"], 0) + class Aggregation(unittest.TestCase): def setUp(self) -> None: @@ -102,6 +119,13 @@ def test_required_failure_is_failed(self) -> None: ) self.assertEqual(report.coverage, SemanticCoverage.FAILED) + def test_required_failure_takes_precedence_over_other_limitations(self) -> None: + report = self.report_for(( + observation("negative", ObservationVerdict.FAIL), + observation("positive", ObservationVerdict.PASS), + )) + self.assertEqual(report.coverage, SemanticCoverage.FAILED) + def test_inconclusive_or_error_required_observations_are_unknown(self) -> None: for verdict in (ObservationVerdict.INCONCLUSIVE, ObservationVerdict.ERROR): with self.subTest(verdict=verdict): @@ -154,20 +178,20 @@ def test_unexpected_or_only_optional_evidence_cannot_verify_coverage(self) -> No observation("unrelated", ObservationVerdict.PASS), ) ) - optional = SemanticProbeRunner().run( - (probe("optional", ProbeKind.NEGATIVE, required=False),), - CONFIG, - RecordedSemanticVerifier((observation("optional", ObservationVerdict.PASS),)), - ) + with self.assertRaises(ValueError): + SemanticProbeRunner().run( + (probe("optional", ProbeKind.NEGATIVE, required=False),), + CONFIG, + RecordedSemanticVerifier((observation("optional", ObservationVerdict.PASS),)), + ) self.assertEqual(unexpected.coverage, SemanticCoverage.UNKNOWN) - self.assertEqual(optional.coverage, SemanticCoverage.UNKNOWN) class SerializationAndPrivacy(unittest.TestCase): def test_report_serialization_is_deterministic(self) -> None: probes = ( - probe("preserve", ProbeKind.PRESERVATION), - probe("negative", ProbeKind.NEGATIVE), + probe("preserve", ProbeKind.PRESERVATION, operation=SemanticOperation.ERASURE), + probe("negative", ProbeKind.NEGATIVE, operation=SemanticOperation.ERASURE), ) observations = ( observation("negative", ObservationVerdict.PASS), @@ -182,17 +206,77 @@ def test_report_serialization_is_deterministic(self) -> None: def test_erasure_report_does_not_disclose_retired_value(self) -> None: erased_value = "orchid-lantern-secret" erasure_probe = probe( - "erase-negative", ProbeKind.NEGATIVE, operation="erase" + "erase-negative", ProbeKind.NEGATIVE, operation=SemanticOperation.ERASURE + ) + preservation_probe = probe( + "erase-preserve", ProbeKind.PRESERVATION, operation=SemanticOperation.ERASURE ) report = SemanticProbeRunner().run( - (erasure_probe,), + (erasure_probe, preservation_probe), CONFIG, RecordedSemanticVerifier( - (observation("erase-negative", ObservationVerdict.PASS),) + ( + observation("erase-negative", ObservationVerdict.PASS), + observation("erase-preserve", ObservationVerdict.PASS), + ) ), ) self.assertNotIn(erased_value, report.canonical_json()) + def test_erasure_rejects_prompt_template_that_can_embed_a_retired_value(self) -> None: + with self.assertRaises(ValueError): + SemanticProbe( + "erase-negative", ProbeKind.NEGATIVE, SemanticOperation.ERASURE, + "current-answer-sample", "Does it remember orchid-lantern-secret?", + ) + + def test_report_parser_rejects_contradictory_coverage(self) -> None: + probes = ( + probe("negative", ProbeKind.NEGATIVE), + probe("positive", ProbeKind.POSITIVE), + probe("preserve", ProbeKind.PRESERVATION), + ) + report = SemanticProbeRunner().run(probes, CONFIG, RecordedSemanticVerifier(( + observation("negative", ObservationVerdict.PASS), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + ))) + payload = report.to_dict() + payload["coverage"] = "failed" + with self.assertRaises(ValueError): + SemanticProbeReport.from_dict(payload) + + +class RequiredTriadAndAdapterBoundaries(unittest.TestCase): + def test_required_triads_and_operation_vocabulary_are_enforced(self) -> None: + runner = SemanticProbeRunner() + with self.assertRaises(ValueError): + runner.run((probe("negative", ProbeKind.NEGATIVE),), CONFIG, ()) + with self.assertRaises(ValueError): + runner.run(( + probe("negative", ProbeKind.NEGATIVE, operation=SemanticOperation.ERASURE), + probe("preserve", ProbeKind.PRESERVATION, operation=SemanticOperation.ERASURE), + probe("positive", ProbeKind.POSITIVE, operation=SemanticOperation.ERASURE), + ), CONFIG, ()) + with self.assertRaises(ValueError): + SemanticProbe("bad", ProbeKind.NEGATIVE, "correct", "scope", "template") + + def test_provider_exception_and_malformed_result_are_unknown(self) -> None: + class Explodes: + def evaluate(self, probe, config): + raise RuntimeError("provider down") + class Malformed: + def evaluate(self, probe, config): + return "pass" + probes = ( + probe("negative", ProbeKind.NEGATIVE), probe("positive", ProbeKind.POSITIVE), + probe("preserve", ProbeKind.PRESERVATION), + ) + for verifier in (Explodes(), Malformed()): + report = SemanticProbeRunner().run(probes, CONFIG, verifier) + self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) + self.assertTrue(report.limitations) + class StrictParsing(unittest.TestCase): def test_observation_parser_rejects_free_form_output_and_bad_digest(self) -> None: From 85eb6d049f3fe047fe591ae35c22de7337713a47 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 16:20:02 +0200 Subject: [PATCH 04/88] fix: close semantic report trust gaps --- prototype/semantic.py | 129 +++++++++++++++++++---------------------- tests/test_semantic.py | 68 ++++++++++++++++++---- 2 files changed, 115 insertions(+), 82 deletions(-) diff --git a/prototype/semantic.py b/prototype/semantic.py index 15eee1a..8b03980 100644 --- a/prototype/semantic.py +++ b/prototype/semantic.py @@ -10,12 +10,15 @@ import hashlib import json +import re from dataclasses import dataclass from datetime import datetime from enum import Enum from types import MappingProxyType from typing import Any, Iterable, Mapping, Protocol, Sequence +from prototype.errata import Operation + def _canonical_json(value: object) -> str: try: @@ -107,15 +110,9 @@ class SemanticCoverage(str, Enum): FAILED = "failed" -class SemanticOperation(str, Enum): - """Canonical operation vocabulary retained by semantic evidence.""" - - CORRECTION = "correction" - SUPERSESSION = "supersession" - ERASURE = "erasure" - - ERASURE_PROMPT_TEMPLATE = "erasure-content-free-v1" +ERASURE_SCOPE = "erasure-scope-v1" +_OPAQUE_TOKEN = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") @dataclass(frozen=True) @@ -129,7 +126,7 @@ class SemanticProbe: probe_id: str kind: ProbeKind - operation: SemanticOperation + operation: Operation scope: str prompt_template: str required: bool = True @@ -138,19 +135,20 @@ def __post_init__(self) -> None: _nonempty_string(self.probe_id, name="probe_id") if not isinstance(self.kind, ProbeKind): raise ValueError("kind must be a ProbeKind") - if not isinstance(self.operation, SemanticOperation): - raise ValueError("operation must be a SemanticOperation") + if not isinstance(self.operation, Operation): + raise ValueError("operation must be an Operation") _nonempty_string(self.scope, name="scope") _nonempty_string(self.prompt_template, name="prompt_template") if not isinstance(self.required, bool): raise ValueError("required must be boolean") - if self.operation is SemanticOperation.ERASURE and self.kind is ProbeKind.POSITIVE: + if self.operation is Operation.ERASE and self.kind is ProbeKind.POSITIVE: raise ValueError("erasure has no positive replacement probe") if ( - self.operation is SemanticOperation.ERASURE - and self.prompt_template != ERASURE_PROMPT_TEMPLATE + self.operation is Operation.ERASE + and (self.prompt_template != ERASURE_PROMPT_TEMPLATE or self.scope != ERASURE_SCOPE + or _OPAQUE_TOKEN.fullmatch(self.probe_id) is None) ): - raise ValueError("erasure probes must use the fixed content-free prompt template") + raise ValueError("erasure probes require fixed content-free fields and opaque token ID") def to_dict(self) -> dict[str, Any]: return { @@ -176,7 +174,7 @@ def from_dict(cls, value: object) -> "SemanticProbe": except (TypeError, ValueError) as error: raise ValueError("semantic probe kind is invalid") from error try: - operation = SemanticOperation(payload["operation"]) + operation = Operation(payload["operation"]) except (TypeError, ValueError) as error: raise ValueError("semantic probe operation is invalid") from error return cls( @@ -389,15 +387,17 @@ def from_dict(cls, value: object) -> "SemanticProbeReport": _nonempty_string(item, name="limitation") for item in payload["limitations"] ), ) - expected = _coverage_for( - report.probes, report.observations, report.config_digest, report.limitations + expected_coverage, expected_limitations = _evaluate_evidence( + report.probes, report.observations, report.config_digest ) - if report.coverage is not expected: + if report.limitations != expected_limitations: + raise ValueError("semantic probe report limitations contradict its evidence") + if report.coverage is not expected_coverage: raise ValueError("semantic probe report coverage contradicts its evidence") return report -def _required_triad(probes: Sequence[SemanticProbe]) -> tuple[SemanticOperation, set[ProbeKind]]: +def _required_triad(probes: Sequence[SemanticProbe]) -> tuple[Operation, set[ProbeKind]]: operations = {probe.operation for probe in probes} if len(operations) != 1: raise ValueError("all probes in a report must have one operation") @@ -405,7 +405,7 @@ def _required_triad(probes: Sequence[SemanticProbe]) -> tuple[SemanticOperation, required_kinds = {probe.kind for probe in probes if probe.required} expected = ( {ProbeKind.NEGATIVE, ProbeKind.PRESERVATION} - if operation is SemanticOperation.ERASURE + if operation is Operation.ERASE else {ProbeKind.NEGATIVE, ProbeKind.POSITIVE, ProbeKind.PRESERVATION} ) if required_kinds != expected: @@ -413,23 +413,44 @@ def _required_triad(probes: Sequence[SemanticProbe]) -> tuple[SemanticOperation, return operation, expected -def _coverage_for( +def _evaluate_evidence( probes: Sequence[SemanticProbe], observations: Sequence[SemanticObservation], config_digest: str, - limitations: Sequence[str], -) -> SemanticCoverage: - required = {probe.probe_id for probe in probes if probe.required} - valid_fails = { - item.probe_id - for item in observations - if item.probe_id in required - and item.config_digest == config_digest - and item.verdict is ObservationVerdict.FAIL - } + *, + initial_limitations: Sequence[str] = (), +) -> tuple[SemanticCoverage, tuple[str, ...]]: + _required_triad(probes) + limitations = list(initial_limitations) + declared = {item.probe_id: item for item in probes} + grouped: dict[str, list[SemanticObservation]] = {} + for item in observations: + grouped.setdefault(item.probe_id, []).append(item) + valid_fails: set[str] = set() + for probe_id, records in grouped.items(): + if probe_id not in declared: + limitations.append(f"unexpected observation for probe {probe_id}") + elif len(records) != 1: + limitations.append(f"duplicate observations for probe {probe_id}") + elif records[0].config_digest != config_digest: + limitations.append(f"configuration drift for probe {probe_id}") + elif records[0].verdict is ObservationVerdict.FAIL and declared[probe_id].required: + valid_fails.add(probe_id) + for probe in probes: + records = grouped.get(probe.probe_id, []) + if not records and probe.required: + limitations.append(f"missing required observation for probe {probe.probe_id}") + elif len(records) == 1 and records[0].config_digest == config_digest: + if probe.required and records[0].verdict in { + ObservationVerdict.INCONCLUSIVE, ObservationVerdict.ERROR + }: + limitations.append( + f"required probe {probe.probe_id} returned {records[0].verdict.value}" + ) + normalized = tuple(sorted(set(limitations))) if valid_fails: - return SemanticCoverage.FAILED - return SemanticCoverage.UNKNOWN if limitations else SemanticCoverage.VERIFIED + return SemanticCoverage.FAILED, normalized + return (SemanticCoverage.UNKNOWN if normalized else SemanticCoverage.VERIFIED), normalized class SemanticProbeRunner: @@ -447,7 +468,6 @@ def run( ids = [item.probe_id for item in declared] if len(ids) != len(set(ids)): raise ValueError("probe IDs must be unique") - _required_triad(declared) adapter_limitations: list[str] = [] if isinstance(verifier, RecordedSemanticVerifier): @@ -479,45 +499,14 @@ def run( raise ValueError("observations must be SemanticObservation instances") limitations: list[str] = list(adapter_limitations) - declared_by_id = {item.probe_id: item for item in declared} - received_by_id: dict[str, list[SemanticObservation]] = {} - for item in received: - received_by_id.setdefault(item.probe_id, []).append(item) - - for probe_id in sorted(received_by_id): - records = received_by_id[probe_id] - if probe_id not in declared_by_id: - limitations.append(f"unexpected observation for probe {probe_id}") - elif len(records) != 1: - limitations.append(f"duplicate observations for probe {probe_id}") - elif records[0].config_digest != config.digest: - limitations.append(f"configuration drift for probe {probe_id}") - - valid: dict[str, SemanticObservation] = {} - for probe in declared: - records = received_by_id.get(probe.probe_id, []) - if not records: - if probe.required: - limitations.append(f"missing required observation for probe {probe.probe_id}") - continue - if len(records) != 1 or records[0].config_digest != config.digest: - continue - record = records[0] - if probe.required and record.verdict in { - ObservationVerdict.INCONCLUSIVE, - ObservationVerdict.ERROR, - }: - limitations.append( - f"required probe {probe.probe_id} returned {record.verdict.value}" - ) - valid[probe.probe_id] = record - - coverage = _coverage_for(declared, received, config.digest, limitations) + coverage, limitations = _evaluate_evidence( + declared, received, config.digest, initial_limitations=limitations + ) return SemanticProbeReport( config_digest=config.digest, coverage=coverage, probes=declared, observations=tuple(received), - limitations=tuple(sorted(set(limitations))), + limitations=limitations, ) diff --git a/tests/test_semantic.py b/tests/test_semantic.py index 81dc7e2..d08cefd 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -13,12 +13,12 @@ RecordedSemanticVerifier, SemanticCoverage, SemanticObservation, - SemanticOperation, SemanticProbe, SemanticProbeReport, SemanticProbeRunner, VerifierConfig, ) +from prototype.errata import Operation CONFIG = VerifierConfig( @@ -31,16 +31,16 @@ def probe( - probe_id: str, kind: ProbeKind, *, required: bool = True, operation: SemanticOperation = SemanticOperation.CORRECTION + probe_id: str, kind: ProbeKind, *, required: bool = True, operation: Operation = Operation.CORRECT ) -> SemanticProbe: return SemanticProbe( probe_id=probe_id, kind=kind, operation=operation, - scope="current-answer-sample", + scope="erasure-scope-v1" if operation is Operation.ERASE else "current-answer-sample", prompt_template=( ERASURE_PROMPT_TEMPLATE - if operation is SemanticOperation.ERASURE + if operation is Operation.ERASE else f"synthetic-{kind.value}-prompt" ), required=required, @@ -126,6 +126,15 @@ def test_required_failure_takes_precedence_over_other_limitations(self) -> None: )) self.assertEqual(report.coverage, SemanticCoverage.FAILED) + def test_duplicate_required_failure_is_invalid_and_unknown(self) -> None: + report = self.report_for(( + observation("negative", ObservationVerdict.FAIL), + observation("negative", ObservationVerdict.PASS), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + )) + self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) + def test_inconclusive_or_error_required_observations_are_unknown(self) -> None: for verdict in (ObservationVerdict.INCONCLUSIVE, ObservationVerdict.ERROR): with self.subTest(verdict=verdict): @@ -190,8 +199,8 @@ def test_unexpected_or_only_optional_evidence_cannot_verify_coverage(self) -> No class SerializationAndPrivacy(unittest.TestCase): def test_report_serialization_is_deterministic(self) -> None: probes = ( - probe("preserve", ProbeKind.PRESERVATION, operation=SemanticOperation.ERASURE), - probe("negative", ProbeKind.NEGATIVE, operation=SemanticOperation.ERASURE), + probe("preserve", ProbeKind.PRESERVATION, operation=Operation.ERASE), + probe("negative", ProbeKind.NEGATIVE, operation=Operation.ERASE), ) observations = ( observation("negative", ObservationVerdict.PASS), @@ -206,10 +215,10 @@ def test_report_serialization_is_deterministic(self) -> None: def test_erasure_report_does_not_disclose_retired_value(self) -> None: erased_value = "orchid-lantern-secret" erasure_probe = probe( - "erase-negative", ProbeKind.NEGATIVE, operation=SemanticOperation.ERASURE + "erase_negative", ProbeKind.NEGATIVE, operation=Operation.ERASE ) preservation_probe = probe( - "erase-preserve", ProbeKind.PRESERVATION, operation=SemanticOperation.ERASURE + "erase_preserve", ProbeKind.PRESERVATION, operation=Operation.ERASE ) report = SemanticProbeRunner().run( (erasure_probe, preservation_probe), @@ -226,10 +235,25 @@ def test_erasure_report_does_not_disclose_retired_value(self) -> None: def test_erasure_rejects_prompt_template_that_can_embed_a_retired_value(self) -> None: with self.assertRaises(ValueError): SemanticProbe( - "erase-negative", ProbeKind.NEGATIVE, SemanticOperation.ERASURE, + "erase-negative", ProbeKind.NEGATIVE, Operation.ERASE, "current-answer-sample", "Does it remember orchid-lantern-secret?", ) + def test_erasure_rejects_retired_prose_in_every_persisted_probe_string(self) -> None: + erased = "orchid lantern secret" + for field in ("probe_id", "scope", "prompt_template"): + values = { + "probe_id": "erase_negative", + "scope": "erasure-scope-v1", + "prompt_template": ERASURE_PROMPT_TEMPLATE, + } + values[field] = erased + with self.subTest(field=field), self.assertRaises(ValueError): + SemanticProbe( + values["probe_id"], ProbeKind.NEGATIVE, Operation.ERASE, + values["scope"], values["prompt_template"], + ) + def test_report_parser_rejects_contradictory_coverage(self) -> None: probes = ( probe("negative", ProbeKind.NEGATIVE), @@ -246,6 +270,26 @@ def test_report_parser_rejects_contradictory_coverage(self) -> None: with self.assertRaises(ValueError): SemanticProbeReport.from_dict(payload) + def test_report_parser_rejects_supplied_limitations_and_missing_triad(self) -> None: + probes = ( + probe("negative", ProbeKind.NEGATIVE), + probe("positive", ProbeKind.POSITIVE), + probe("preserve", ProbeKind.PRESERVATION), + ) + report = SemanticProbeRunner().run(probes, CONFIG, RecordedSemanticVerifier(( + observation("negative", ObservationVerdict.PASS), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + ))) + payload = report.to_dict() + payload["limitations"] = ["invented limitation"] + with self.assertRaises(ValueError): + SemanticProbeReport.from_dict(payload) + payload = report.to_dict() + payload["probes"].pop() + with self.assertRaises(ValueError): + SemanticProbeReport.from_dict(payload) + class RequiredTriadAndAdapterBoundaries(unittest.TestCase): def test_required_triads_and_operation_vocabulary_are_enforced(self) -> None: @@ -254,9 +298,9 @@ def test_required_triads_and_operation_vocabulary_are_enforced(self) -> None: runner.run((probe("negative", ProbeKind.NEGATIVE),), CONFIG, ()) with self.assertRaises(ValueError): runner.run(( - probe("negative", ProbeKind.NEGATIVE, operation=SemanticOperation.ERASURE), - probe("preserve", ProbeKind.PRESERVATION, operation=SemanticOperation.ERASURE), - probe("positive", ProbeKind.POSITIVE, operation=SemanticOperation.ERASURE), + probe("negative", ProbeKind.NEGATIVE, operation=Operation.ERASE), + probe("preserve", ProbeKind.PRESERVATION, operation=Operation.ERASE), + probe("positive", ProbeKind.POSITIVE, operation=Operation.ERASE), ), CONFIG, ()) with self.assertRaises(ValueError): SemanticProbe("bad", ProbeKind.NEGATIVE, "correct", "scope", "template") From 6ef71b4ab7dd2b685d07ed300b39084bd7cb85e2 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 16:23:44 +0200 Subject: [PATCH 05/88] fix: make erasure evidence content free --- prototype/semantic.py | 69 ++++++++++++++++++++++++++++++------------ tests/test_semantic.py | 39 ++++++++++++++++++++---- 2 files changed, 82 insertions(+), 26 deletions(-) diff --git a/prototype/semantic.py b/prototype/semantic.py index 8b03980..529b0da 100644 --- a/prototype/semantic.py +++ b/prototype/semantic.py @@ -49,6 +49,21 @@ def _nonempty_string(value: object, *, name: str) -> str: return value +def _probe_id(value: object, *, name: str) -> str: + text = _nonempty_string(value, name=name) + if _PROBE_ID.fullmatch(text) is None: + raise ValueError(f"{name} must be a safe protocol identifier") + return text + + +def _erasure_probe_id(kind: ProbeKind) -> str: + if kind is ProbeKind.NEGATIVE: + return ERASURE_NEGATIVE_PROBE_ID + if kind is ProbeKind.PRESERVATION: + return ERASURE_PRESERVATION_PROBE_ID + raise ValueError("erasure has no positive replacement probe") + + def _sha256(value: object, *, name: str) -> str: text = _nonempty_string(value, name=name) if len(text) != 64 or any(char not in "0123456789abcdef" for char in text): @@ -111,8 +126,10 @@ class SemanticCoverage(str, Enum): ERASURE_PROMPT_TEMPLATE = "erasure-content-free-v1" -ERASURE_SCOPE = "erasure-scope-v1" -_OPAQUE_TOKEN = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") +ERASURE_SCOPE = "declared-store-set-v1" +ERASURE_NEGATIVE_PROBE_ID = "erase-negative-v1" +ERASURE_PRESERVATION_PROBE_ID = "erase-preservation-v1" +_PROBE_ID = re.compile(r"[a-z][a-z0-9-]{0,63}\Z") @dataclass(frozen=True) @@ -132,7 +149,7 @@ class SemanticProbe: required: bool = True def __post_init__(self) -> None: - _nonempty_string(self.probe_id, name="probe_id") + _probe_id(self.probe_id, name="probe_id") if not isinstance(self.kind, ProbeKind): raise ValueError("kind must be a ProbeKind") if not isinstance(self.operation, Operation): @@ -145,10 +162,13 @@ def __post_init__(self) -> None: raise ValueError("erasure has no positive replacement probe") if ( self.operation is Operation.ERASE - and (self.prompt_template != ERASURE_PROMPT_TEMPLATE or self.scope != ERASURE_SCOPE - or _OPAQUE_TOKEN.fullmatch(self.probe_id) is None) + and ( + self.prompt_template != ERASURE_PROMPT_TEMPLATE + or self.scope != ERASURE_SCOPE + or self.probe_id != _erasure_probe_id(self.kind) + ) ): - raise ValueError("erasure probes require fixed content-free fields and opaque token ID") + raise ValueError("erasure probes require fixed content-free protocol fields") def to_dict(self) -> dict[str, Any]: return { @@ -178,7 +198,7 @@ def from_dict(cls, value: object) -> "SemanticProbe": except (TypeError, ValueError) as error: raise ValueError("semantic probe operation is invalid") from error return cls( - probe_id=_nonempty_string(payload["probe_id"], name="probe_id"), + probe_id=_probe_id(payload["probe_id"], name="probe_id"), kind=kind, operation=operation, scope=_nonempty_string(payload["scope"], name="scope"), @@ -255,7 +275,7 @@ class SemanticObservation: response_digest: str def __post_init__(self) -> None: - _nonempty_string(self.probe_id, name="probe_id") + _probe_id(self.probe_id, name="probe_id") if not isinstance(self.verdict, ObservationVerdict): raise ValueError("verdict must be an ObservationVerdict") _sha256(self.config_digest, name="config_digest") @@ -285,7 +305,7 @@ def from_dict(cls, value: object) -> "SemanticObservation": except (TypeError, ValueError) as error: raise ValueError("semantic observation verdict is invalid") from error return cls( - probe_id=_nonempty_string(payload["probe_id"], name="probe_id"), + probe_id=_probe_id(payload["probe_id"], name="probe_id"), verdict=verdict, config_digest=_sha256(payload["config_digest"], name="config_digest"), observed_at=_timestamp(payload["observed_at"], name="observed_at"), @@ -387,9 +407,11 @@ def from_dict(cls, value: object) -> "SemanticProbeReport": _nonempty_string(item, name="limitation") for item in payload["limitations"] ), ) - expected_coverage, expected_limitations = _evaluate_evidence( + expected_coverage, expected_limitations, accepted = _evaluate_evidence( report.probes, report.observations, report.config_digest ) + if report.observations != accepted: + raise ValueError("semantic probe report persists unexpected observations") if report.limitations != expected_limitations: raise ValueError("semantic probe report limitations contradict its evidence") if report.coverage is not expected_coverage: @@ -419,7 +441,7 @@ def _evaluate_evidence( config_digest: str, *, initial_limitations: Sequence[str] = (), -) -> tuple[SemanticCoverage, tuple[str, ...]]: +) -> tuple[SemanticCoverage, tuple[str, ...], tuple[SemanticObservation, ...]]: _required_triad(probes) limitations = list(initial_limitations) declared = {item.probe_id: item for item in probes} @@ -427,15 +449,21 @@ def _evaluate_evidence( for item in observations: grouped.setdefault(item.probe_id, []).append(item) valid_fails: set[str] = set() + accepted: list[SemanticObservation] = [] + unexpected_count = 0 for probe_id, records in grouped.items(): if probe_id not in declared: - limitations.append(f"unexpected observation for probe {probe_id}") + unexpected_count += len(records) elif len(records) != 1: limitations.append(f"duplicate observations for probe {probe_id}") - elif records[0].config_digest != config_digest: - limitations.append(f"configuration drift for probe {probe_id}") - elif records[0].verdict is ObservationVerdict.FAIL and declared[probe_id].required: - valid_fails.add(probe_id) + else: + accepted.append(records[0]) + if records[0].config_digest != config_digest: + limitations.append(f"configuration drift for probe {probe_id}") + elif records[0].verdict is ObservationVerdict.FAIL and declared[probe_id].required: + valid_fails.add(probe_id) + if unexpected_count: + limitations.append(f"unexpected observation identifiers: {unexpected_count}") for probe in probes: records = grouped.get(probe.probe_id, []) if not records and probe.required: @@ -448,9 +476,10 @@ def _evaluate_evidence( f"required probe {probe.probe_id} returned {records[0].verdict.value}" ) normalized = tuple(sorted(set(limitations))) + persisted = tuple(sorted(accepted, key=lambda item: (item.probe_id, item.observed_at, item.response_digest))) if valid_fails: - return SemanticCoverage.FAILED, normalized - return (SemanticCoverage.UNKNOWN if normalized else SemanticCoverage.VERIFIED), normalized + return SemanticCoverage.FAILED, normalized, persisted + return (SemanticCoverage.UNKNOWN if normalized else SemanticCoverage.VERIFIED), normalized, persisted class SemanticProbeRunner: @@ -499,7 +528,7 @@ def run( raise ValueError("observations must be SemanticObservation instances") limitations: list[str] = list(adapter_limitations) - coverage, limitations = _evaluate_evidence( + coverage, limitations, persisted = _evaluate_evidence( declared, received, config.digest, initial_limitations=limitations ) @@ -507,6 +536,6 @@ def run( config_digest=config.digest, coverage=coverage, probes=declared, - observations=tuple(received), + observations=persisted, limitations=limitations, ) diff --git a/tests/test_semantic.py b/tests/test_semantic.py index d08cefd..67705b4 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -8,6 +8,8 @@ from prototype.semantic import ( ObservationVerdict, + ERASURE_NEGATIVE_PROBE_ID, + ERASURE_PRESERVATION_PROBE_ID, ERASURE_PROMPT_TEMPLATE, ProbeKind, RecordedSemanticVerifier, @@ -34,10 +36,12 @@ def probe( probe_id: str, kind: ProbeKind, *, required: bool = True, operation: Operation = Operation.CORRECT ) -> SemanticProbe: return SemanticProbe( - probe_id=probe_id, + probe_id=( + ERASURE_NEGATIVE_PROBE_ID if kind is ProbeKind.NEGATIVE else ERASURE_PRESERVATION_PROBE_ID + ) if operation is Operation.ERASE else probe_id, kind=kind, operation=operation, - scope="erasure-scope-v1" if operation is Operation.ERASE else "current-answer-sample", + scope="declared-store-set-v1" if operation is Operation.ERASE else "current-answer-sample", prompt_template=( ERASURE_PROMPT_TEMPLATE if operation is Operation.ERASE @@ -215,18 +219,18 @@ def test_report_serialization_is_deterministic(self) -> None: def test_erasure_report_does_not_disclose_retired_value(self) -> None: erased_value = "orchid-lantern-secret" erasure_probe = probe( - "erase_negative", ProbeKind.NEGATIVE, operation=Operation.ERASE + ERASURE_NEGATIVE_PROBE_ID, ProbeKind.NEGATIVE, operation=Operation.ERASE ) preservation_probe = probe( - "erase_preserve", ProbeKind.PRESERVATION, operation=Operation.ERASE + ERASURE_PRESERVATION_PROBE_ID, ProbeKind.PRESERVATION, operation=Operation.ERASE ) report = SemanticProbeRunner().run( (erasure_probe, preservation_probe), CONFIG, RecordedSemanticVerifier( ( - observation("erase-negative", ObservationVerdict.PASS), - observation("erase-preserve", ObservationVerdict.PASS), + observation(ERASURE_NEGATIVE_PROBE_ID, ObservationVerdict.PASS), + observation(ERASURE_PRESERVATION_PROBE_ID, ObservationVerdict.PASS), ) ), ) @@ -254,6 +258,13 @@ def test_erasure_rejects_retired_prose_in_every_persisted_probe_string(self) -> values["scope"], values["prompt_template"], ) + def test_erasure_rejects_caller_selected_protocol_identifiers(self) -> None: + with self.assertRaises(ValueError): + SemanticProbe( + "other-safe-token", ProbeKind.NEGATIVE, Operation.ERASE, + "declared-store-set-v1", ERASURE_PROMPT_TEMPLATE, + ) + def test_report_parser_rejects_contradictory_coverage(self) -> None: probes = ( probe("negative", ProbeKind.NEGATIVE), @@ -270,6 +281,22 @@ def test_report_parser_rejects_contradictory_coverage(self) -> None: with self.assertRaises(ValueError): SemanticProbeReport.from_dict(payload) + def test_unexpected_erasure_observation_identifier_is_not_persisted(self) -> None: + retired = "orchid-lantern-secret" + probes = ( + probe("ignored", ProbeKind.NEGATIVE, operation=Operation.ERASE), + probe("ignored", ProbeKind.PRESERVATION, operation=Operation.ERASE), + ) + report = SemanticProbeRunner().run(probes, CONFIG, RecordedSemanticVerifier(( + observation(ERASURE_NEGATIVE_PROBE_ID, ObservationVerdict.PASS), + observation(ERASURE_PRESERVATION_PROBE_ID, ObservationVerdict.PASS), + observation(retired, ObservationVerdict.PASS), + ))) + self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) + self.assertEqual(len(report.observations), 2) + self.assertIn("unexpected observation identifiers: 1", report.limitations) + self.assertNotIn(retired, report.canonical_json()) + def test_report_parser_rejects_supplied_limitations_and_missing_triad(self) -> None: probes = ( probe("negative", ProbeKind.NEGATIVE), From dff1aec3494704bb53b4b1b2a9da30a3e48ec7f2 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 16:34:28 +0200 Subject: [PATCH 06/88] fix: bind semantic observations to operations --- prototype/semantic.py | 89 +++++++++++++++++++++------ tests/test_semantic.py | 135 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 198 insertions(+), 26 deletions(-) diff --git a/prototype/semantic.py b/prototype/semantic.py index 529b0da..5ee95fc 100644 --- a/prototype/semantic.py +++ b/prototype/semantic.py @@ -269,6 +269,7 @@ class SemanticObservation: """Persistable verifier result, excluding raw provider output.""" probe_id: str + operation: Operation verdict: ObservationVerdict config_digest: str observed_at: str @@ -276,6 +277,14 @@ class SemanticObservation: def __post_init__(self) -> None: _probe_id(self.probe_id, name="probe_id") + if not isinstance(self.operation, Operation): + raise ValueError("operation must be an Operation") + if ( + self.operation is Operation.ERASE + and self.probe_id + not in {ERASURE_NEGATIVE_PROBE_ID, ERASURE_PRESERVATION_PROBE_ID} + ): + raise ValueError("erasure observations require fixed content-free probe identifiers") if not isinstance(self.verdict, ObservationVerdict): raise ValueError("verdict must be an ObservationVerdict") _sha256(self.config_digest, name="config_digest") @@ -285,6 +294,7 @@ def __post_init__(self) -> None: def to_dict(self) -> dict[str, str]: return { "probe_id": self.probe_id, + "operation": self.operation.value, "verdict": self.verdict.value, "config_digest": self.config_digest, "observed_at": self.observed_at, @@ -296,16 +306,28 @@ def from_dict(cls, value: object) -> "SemanticObservation": payload = _exact_fields( value, fields=frozenset( - {"probe_id", "verdict", "config_digest", "observed_at", "response_digest"} + { + "probe_id", + "operation", + "verdict", + "config_digest", + "observed_at", + "response_digest", + } ), name="semantic observation", ) + try: + operation = Operation(payload["operation"]) + except (TypeError, ValueError) as error: + raise ValueError("semantic observation operation is invalid") from error try: verdict = ObservationVerdict(payload["verdict"]) except (TypeError, ValueError) as error: raise ValueError("semantic observation verdict is invalid") from error return cls( probe_id=_probe_id(payload["probe_id"], name="probe_id"), + operation=operation, verdict=verdict, config_digest=_sha256(payload["config_digest"], name="config_digest"), observed_at=_timestamp(payload["observed_at"], name="observed_at"), @@ -336,7 +358,13 @@ def observations(self) -> tuple[SemanticObservation, ...]: def evaluate( self, probe: SemanticProbe, config: VerifierConfig ) -> SemanticObservation | None: - matches = [item for item in self._observations if item.probe_id == probe.probe_id] + matches = [ + item + for item in self._observations + if item.probe_id == probe.probe_id + and item.operation is probe.operation + and item.config_digest == config.digest + ] return matches[0] if len(matches) == 1 else None @@ -360,6 +388,21 @@ def __post_init__(self) -> None: raise ValueError("observations must be SemanticObservation instances") if not all(isinstance(item, str) and item for item in self.limitations): raise ValueError("limitations must contain non-empty strings") + expected_coverage, expected_limitations, accepted = _evaluate_evidence( + self.probes, + self.observations, + self.config_digest, + initial_limitations=self.limitations, + ) + if self.observations != accepted: + raise ValueError( + "semantic probe report observations must uniquely match declared probes, " + "operations, and configuration" + ) + if self.limitations != expected_limitations: + raise ValueError("semantic probe report limitations contradict its evidence") + if self.coverage is not expected_coverage: + raise ValueError("semantic probe report coverage contradicts its evidence") def to_dict(self) -> dict[str, Any]: return { @@ -407,19 +450,15 @@ def from_dict(cls, value: object) -> "SemanticProbeReport": _nonempty_string(item, name="limitation") for item in payload["limitations"] ), ) - expected_coverage, expected_limitations, accepted = _evaluate_evidence( - report.probes, report.observations, report.config_digest - ) - if report.observations != accepted: - raise ValueError("semantic probe report persists unexpected observations") - if report.limitations != expected_limitations: - raise ValueError("semantic probe report limitations contradict its evidence") - if report.coverage is not expected_coverage: - raise ValueError("semantic probe report coverage contradicts its evidence") return report def _required_triad(probes: Sequence[SemanticProbe]) -> tuple[Operation, set[ProbeKind]]: + if not probes or not all(isinstance(probe, SemanticProbe) for probe in probes): + raise ValueError("probes must be SemanticProbe instances") + probe_ids = [probe.probe_id for probe in probes] + if len(probe_ids) != len(set(probe_ids)): + raise ValueError("probe IDs must be unique") operations = {probe.operation for probe in probes} if len(operations) != 1: raise ValueError("all probes in a report must have one operation") @@ -450,6 +489,7 @@ def _evaluate_evidence( grouped.setdefault(item.probe_id, []).append(item) valid_fails: set[str] = set() accepted: list[SemanticObservation] = [] + accepted_by_id: dict[str, SemanticObservation] = {} unexpected_count = 0 for probe_id, records in grouped.items(): if probe_id not in declared: @@ -457,23 +497,34 @@ def _evaluate_evidence( elif len(records) != 1: limitations.append(f"duplicate observations for probe {probe_id}") else: - accepted.append(records[0]) - if records[0].config_digest != config_digest: + record = records[0] + probe = declared[probe_id] + if record.operation is not probe.operation: + limitations.append(f"operation mismatch for probe {probe_id}") + elif record.config_digest != config_digest: limitations.append(f"configuration drift for probe {probe_id}") - elif records[0].verdict is ObservationVerdict.FAIL and declared[probe_id].required: + else: + accepted.append(record) + accepted_by_id[probe_id] = record + if ( + record.operation is probe.operation + and record.config_digest == config_digest + and record.verdict is ObservationVerdict.FAIL + and probe.required + ): valid_fails.add(probe_id) if unexpected_count: limitations.append(f"unexpected observation identifiers: {unexpected_count}") for probe in probes: - records = grouped.get(probe.probe_id, []) - if not records and probe.required: + record = accepted_by_id.get(probe.probe_id) + if record is None and probe.required: limitations.append(f"missing required observation for probe {probe.probe_id}") - elif len(records) == 1 and records[0].config_digest == config_digest: - if probe.required and records[0].verdict in { + elif record is not None: + if probe.required and record.verdict in { ObservationVerdict.INCONCLUSIVE, ObservationVerdict.ERROR }: limitations.append( - f"required probe {probe.probe_id} returned {records[0].verdict.value}" + f"required probe {probe.probe_id} returned {record.verdict.value}" ) normalized = tuple(sorted(set(limitations))) persisted = tuple(sorted(accepted, key=lambda item: (item.probe_id, item.observed_at, item.response_digest))) diff --git a/tests/test_semantic.py b/tests/test_semantic.py index 67705b4..0a8d12e 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -52,10 +52,15 @@ def probe( def observation( - probe_id: str, verdict: ObservationVerdict, *, config: VerifierConfig = CONFIG + probe_id: str, + verdict: ObservationVerdict, + *, + config: VerifierConfig = CONFIG, + operation: Operation = Operation.CORRECT, ) -> SemanticObservation: return SemanticObservation( probe_id=probe_id, + operation=operation, verdict=verdict, config_digest=config.digest, observed_at="2026-08-12T09:00:00Z", @@ -207,8 +212,16 @@ def test_report_serialization_is_deterministic(self) -> None: probe("negative", ProbeKind.NEGATIVE, operation=Operation.ERASE), ) observations = ( - observation("negative", ObservationVerdict.PASS), - observation("preserve", ObservationVerdict.PASS), + observation( + ERASURE_NEGATIVE_PROBE_ID, + ObservationVerdict.PASS, + operation=Operation.ERASE, + ), + observation( + ERASURE_PRESERVATION_PROBE_ID, + ObservationVerdict.PASS, + operation=Operation.ERASE, + ), ) report = SemanticProbeRunner().run(probes, CONFIG, RecordedSemanticVerifier(observations)) self.assertEqual( @@ -229,11 +242,20 @@ def test_erasure_report_does_not_disclose_retired_value(self) -> None: CONFIG, RecordedSemanticVerifier( ( - observation(ERASURE_NEGATIVE_PROBE_ID, ObservationVerdict.PASS), - observation(ERASURE_PRESERVATION_PROBE_ID, ObservationVerdict.PASS), + observation( + ERASURE_NEGATIVE_PROBE_ID, + ObservationVerdict.PASS, + operation=Operation.ERASE, + ), + observation( + ERASURE_PRESERVATION_PROBE_ID, + ObservationVerdict.PASS, + operation=Operation.ERASE, + ), ) ), ) + self.assertEqual(report.coverage, SemanticCoverage.VERIFIED) self.assertNotIn(erased_value, report.canonical_json()) def test_erasure_rejects_prompt_template_that_can_embed_a_retired_value(self) -> None: @@ -288,8 +310,16 @@ def test_unexpected_erasure_observation_identifier_is_not_persisted(self) -> Non probe("ignored", ProbeKind.PRESERVATION, operation=Operation.ERASE), ) report = SemanticProbeRunner().run(probes, CONFIG, RecordedSemanticVerifier(( - observation(ERASURE_NEGATIVE_PROBE_ID, ObservationVerdict.PASS), - observation(ERASURE_PRESERVATION_PROBE_ID, ObservationVerdict.PASS), + observation( + ERASURE_NEGATIVE_PROBE_ID, + ObservationVerdict.PASS, + operation=Operation.ERASE, + ), + observation( + ERASURE_PRESERVATION_PROBE_ID, + ObservationVerdict.PASS, + operation=Operation.ERASE, + ), observation(retired, ObservationVerdict.PASS), ))) self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) @@ -348,6 +378,97 @@ def evaluate(self, probe, config): self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) self.assertTrue(report.limitations) + def test_recorded_observation_operation_mismatch_is_unknown_and_not_persisted(self) -> None: + probes = ( + probe("negative", ProbeKind.NEGATIVE), + probe("positive", ProbeKind.POSITIVE), + probe("preserve", ProbeKind.PRESERVATION), + ) + report = SemanticProbeRunner().run( + probes, + CONFIG, + RecordedSemanticVerifier(( + observation("negative", ObservationVerdict.PASS, operation=Operation.SUPERSEDE), + observation("positive", ObservationVerdict.PASS), + observation("preserve", ObservationVerdict.PASS), + )), + ) + self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) + self.assertNotIn("negative", {item.probe_id for item in report.observations}) + self.assertIn("operation mismatch for probe negative", report.limitations) + + +class DirectConstructionBoundaries(unittest.TestCase): + def setUp(self) -> None: + self.probes = ( + probe("negative", ProbeKind.NEGATIVE), + probe("positive", ProbeKind.POSITIVE), + probe("preserve", ProbeKind.PRESERVATION), + ) + self.observations = tuple( + observation(item.probe_id, ObservationVerdict.PASS) for item in self.probes + ) + + def report_with(self, observations: tuple[SemanticObservation, ...]) -> SemanticProbeReport: + return SemanticProbeReport( + config_digest=CONFIG.digest, + coverage=SemanticCoverage.VERIFIED, + probes=self.probes, + observations=observations, + limitations=(), + ) + + def test_erasure_observation_rejects_caller_selected_identifier(self) -> None: + with self.assertRaises(ValueError): + observation( + "orchid-lantern-secret", + ObservationVerdict.PASS, + operation=Operation.ERASE, + ) + + payload = observation( + ERASURE_NEGATIVE_PROBE_ID, + ObservationVerdict.PASS, + operation=Operation.ERASE, + ).to_dict() + payload["probe_id"] = "orchid-lantern-secret" + with self.assertRaises(ValueError): + SemanticObservation.from_dict(payload) + + def test_observation_serialization_binds_canonical_operation(self) -> None: + item = observation("negative", ObservationVerdict.PASS) + self.assertEqual(item.to_dict()["operation"], Operation.CORRECT.value) + payload = item.to_dict() + payload["operation"] = "correction" + with self.assertRaises(ValueError): + SemanticObservation.from_dict(payload) + + def test_report_rejects_unexpected_duplicate_or_mismatched_observations(self) -> None: + drifted = VerifierConfig( + provider="other-provider", + model="synthetic-model-1", + prompt_template_version="2026-08-12", + sampling={"temperature": 0}, + ) + invalid = { + "unexpected": self.observations + + (observation("unrelated", ObservationVerdict.PASS),), + "duplicate": self.observations + (self.observations[0],), + "operation": ( + observation( + "negative", ObservationVerdict.PASS, operation=Operation.SUPERSEDE + ), + ) + + self.observations[1:], + "configuration": ( + observation("negative", ObservationVerdict.PASS, config=drifted), + ) + + self.observations[1:], + } + for name, records in invalid.items(): + with self.subTest(case=name), self.assertRaises(ValueError): + self.report_with(records) + class StrictParsing(unittest.TestCase): def test_observation_parser_rejects_free_form_output_and_bad_digest(self) -> None: From f53bcf5321f20193ef8f495a4bc5d287cc6204d7 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 16:42:18 +0200 Subject: [PATCH 07/88] fix: derive semantic report limitations --- prototype/semantic.py | 74 ++++++++++++++++++++++++++++++++++-------- tests/test_semantic.py | 11 +++++++ 2 files changed, 71 insertions(+), 14 deletions(-) diff --git a/prototype/semantic.py b/prototype/semantic.py index 5ee95fc..2f0c340 100644 --- a/prototype/semantic.py +++ b/prototype/semantic.py @@ -379,6 +379,9 @@ class SemanticProbeReport: limitations: tuple[str, ...] def __post_init__(self) -> None: + self._validate() + + def _validate(self, *, adapter_diagnostics: Sequence[str] = ()) -> None: _sha256(self.config_digest, name="config_digest") if not isinstance(self.coverage, SemanticCoverage): raise ValueError("coverage must be a SemanticCoverage") @@ -388,11 +391,22 @@ def __post_init__(self) -> None: raise ValueError("observations must be SemanticObservation instances") if not all(isinstance(item, str) and item for item in self.limitations): raise ValueError("limitations must contain non-empty strings") - expected_coverage, expected_limitations, accepted = _evaluate_evidence( + if not all(isinstance(item, str) and item for item in adapter_diagnostics): + raise ValueError("adapter diagnostics must contain non-empty strings") + evidence_coverage, evidence_limitations, accepted = _evaluate_evidence( self.probes, self.observations, self.config_digest, - initial_limitations=self.limitations, + ) + expected_limitations = tuple( + sorted(set(evidence_limitations).union(adapter_diagnostics)) + ) + expected_coverage = ( + SemanticCoverage.FAILED + if evidence_coverage is SemanticCoverage.FAILED + else SemanticCoverage.UNKNOWN + if expected_limitations + else SemanticCoverage.VERIFIED ) if self.observations != accepted: raise ValueError( @@ -404,6 +418,46 @@ def __post_init__(self) -> None: if self.coverage is not expected_coverage: raise ValueError("semantic probe report coverage contradicts its evidence") + @classmethod + def _from_runner_evidence( + cls, + *, + config_digest: str, + probes: Sequence[SemanticProbe], + observations: Sequence[SemanticObservation], + adapter_diagnostics: Sequence[str], + ) -> "SemanticProbeReport": + """Build a report from trusted runner diagnostics and serialized evidence.""" + + evidence_coverage, evidence_limitations, accepted = _evaluate_evidence( + probes, observations, config_digest + ) + # Evidence rejected before persistence (duplicates, unexpected IDs, + # operation/configuration mismatches) cannot be rediscovered by + # validating the accepted observation subset. Carry only the + # evaluator-produced, sanitized messages through this private runner + # path; public construction and parsing still derive their limitations + # exclusively from the serialized evidence they receive. + diagnostics = tuple( + sorted(set(adapter_diagnostics).union(evidence_limitations)) + ) + limitations = tuple(sorted(set(evidence_limitations).union(diagnostics))) + coverage = ( + SemanticCoverage.FAILED + if evidence_coverage is SemanticCoverage.FAILED + else SemanticCoverage.UNKNOWN + if limitations + else SemanticCoverage.VERIFIED + ) + report = object.__new__(cls) + object.__setattr__(report, "config_digest", config_digest) + object.__setattr__(report, "coverage", coverage) + object.__setattr__(report, "probes", tuple(probes)) + object.__setattr__(report, "observations", accepted) + object.__setattr__(report, "limitations", limitations) + report._validate(adapter_diagnostics=diagnostics) + return report + def to_dict(self) -> dict[str, Any]: return { "config_digest": self.config_digest, @@ -478,11 +532,9 @@ def _evaluate_evidence( probes: Sequence[SemanticProbe], observations: Sequence[SemanticObservation], config_digest: str, - *, - initial_limitations: Sequence[str] = (), ) -> tuple[SemanticCoverage, tuple[str, ...], tuple[SemanticObservation, ...]]: _required_triad(probes) - limitations = list(initial_limitations) + limitations: list[str] = [] declared = {item.probe_id: item for item in probes} grouped: dict[str, list[SemanticObservation]] = {} for item in observations: @@ -578,15 +630,9 @@ def run( if not all(isinstance(item, SemanticObservation) for item in received): raise ValueError("observations must be SemanticObservation instances") - limitations: list[str] = list(adapter_limitations) - coverage, limitations, persisted = _evaluate_evidence( - declared, received, config.digest, initial_limitations=limitations - ) - - return SemanticProbeReport( + return SemanticProbeReport._from_runner_evidence( config_digest=config.digest, - coverage=coverage, probes=declared, - observations=persisted, - limitations=limitations, + observations=received, + adapter_diagnostics=adapter_limitations, ) diff --git a/tests/test_semantic.py b/tests/test_semantic.py index 0a8d12e..0295432 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -339,6 +339,7 @@ def test_report_parser_rejects_supplied_limitations_and_missing_triad(self) -> N observation("preserve", ObservationVerdict.PASS), ))) payload = report.to_dict() + payload["coverage"] = "unknown" payload["limitations"] = ["invented limitation"] with self.assertRaises(ValueError): SemanticProbeReport.from_dict(payload) @@ -418,6 +419,16 @@ def report_with(self, observations: tuple[SemanticObservation, ...]) -> Semantic limitations=(), ) + def test_report_rejects_caller_supplied_limitation_that_forces_unknown(self) -> None: + with self.assertRaises(ValueError): + SemanticProbeReport( + config_digest=CONFIG.digest, + coverage=SemanticCoverage.UNKNOWN, + probes=self.probes, + observations=self.observations, + limitations=("invented limitation",), + ) + def test_erasure_observation_rejects_caller_selected_identifier(self) -> None: with self.assertRaises(ValueError): observation( From caf491b2f6d85898e82a0a594c91371b88ac57c3 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 16:53:22 +0200 Subject: [PATCH 08/88] fix: keep provider diagnostics content free --- prototype/semantic.py | 8 ++++++-- tests/test_semantic.py | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/prototype/semantic.py b/prototype/semantic.py index 2f0c340..3f0780e 100644 --- a/prototype/semantic.py +++ b/prototype/semantic.py @@ -610,9 +610,13 @@ def run( for item in declared: try: result = verifier.evaluate(item, config) # type: ignore[union-attr] - except Exception as error: + except Exception: + # Provider exceptions are untrusted input. Their messages + # and even dynamically created class names can contain + # retired or confidential content, so persist only this + # closed diagnostic code plus the declared protocol ID. adapter_limitations.append( - f"provider error for probe {item.probe_id}: {type(error).__name__}" + f"provider error for probe {item.probe_id}" ) continue if result is None: diff --git a/tests/test_semantic.py b/tests/test_semantic.py index 0295432..4335305 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -379,6 +379,32 @@ def evaluate(self, probe, config): self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) self.assertTrue(report.limitations) + def test_provider_exception_type_is_not_persisted_for_erasure(self) -> None: + retired_value = "orchid-lantern-secret" + value_bearing_error = type(retired_value, (Exception,), {}) + + class Explodes: + def evaluate(self, probe, config): + raise value_bearing_error("provider failure") + + probes = ( + probe("ignored", ProbeKind.NEGATIVE, operation=Operation.ERASE), + probe("ignored", ProbeKind.PRESERVATION, operation=Operation.ERASE), + ) + report = SemanticProbeRunner().run(probes, CONFIG, Explodes()) + + self.assertEqual(report.coverage, SemanticCoverage.UNKNOWN) + self.assertEqual( + report.limitations, + ( + f"missing required observation for probe {ERASURE_NEGATIVE_PROBE_ID}", + f"missing required observation for probe {ERASURE_PRESERVATION_PROBE_ID}", + f"provider error for probe {ERASURE_NEGATIVE_PROBE_ID}", + f"provider error for probe {ERASURE_PRESERVATION_PROBE_ID}", + ), + ) + self.assertNotIn(retired_value, report.canonical_json()) + def test_recorded_observation_operation_mismatch_is_unknown_and_not_persisted(self) -> None: probes = ( probe("negative", ProbeKind.NEGATIVE), From 2386fe9bcec450cff9f1aa0a2f155960e0e94d16 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 17:04:01 +0200 Subject: [PATCH 09/88] feat: publish semantic probe conformance fixtures --- .../task-2-report.md | 30 ++++++++ prototype/README.md | 26 +++++++ prototype/cli.py | 74 ++++++++++++++++++- spec/README.md | 34 +++++++++ spec/semantic/observations.json | 74 +++++++++++++++++++ spec/semantic/probes.json | 74 +++++++++++++++++++ spec/semantic/verifier-config.json | 9 +++ tests/test_cli.py | 46 ++++++++++++ 8 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 .superpowers/sdd/2026-08-12-semantic-probes-and-publication/task-2-report.md create mode 100644 spec/semantic/observations.json create mode 100644 spec/semantic/probes.json create mode 100644 spec/semantic/verifier-config.json diff --git a/.superpowers/sdd/2026-08-12-semantic-probes-and-publication/task-2-report.md b/.superpowers/sdd/2026-08-12-semantic-probes-and-publication/task-2-report.md new file mode 100644 index 0000000..89a0331 --- /dev/null +++ b/.superpowers/sdd/2026-08-12-semantic-probes-and-publication/task-2-report.md @@ -0,0 +1,30 @@ +# Task 2 report + +## Loop state — revision 1 + +- Goal: publish offline semantic-probe conformance fixtures and `errata semantic-test`. +- Success measure: verified, failed, and unknown fixtures return `0`, `1`, and + `2`; malformed input returns `1` without traceback; focused tests, full suite, + and `make check` pass. +- Invariants: standard library only; synthetic data only; erasure records remain + content-free; canonical JSON report; no Phase 1 receipt behavior changes. +- Authority: modify only Task 2-owned files in this worktree; preserve unrelated + concurrent work. +- Primary loop: TDD. Rejected: debugging (no unexplained failure), multi-agent + checker (task scope is cohesive and deterministic). +- Budget: three vertical test/implementation slices, one full validation run, + one diff review. Stop and report if Task 1 interface conflicts or a named gate + remains red. +- Gates: PASS — CLI exit behavior (`verified=0`, `failed=1`, `unknown=2`); + PASS — malformed JSON returns `1` without traceback; PASS — canonical JSON, + operation binding, and content-free erasure evidence; PASS — focused tests, + full suite, and `make check`. +- Evidence: `python3 -m unittest tests.test_cli tests.test_semantic -v` (43 + passed); `python3 -m unittest discover -s tests -t .` (210 passed); `make + check` (repository, claim, readiness, 210 tests, and intentional demo exit-2 + gate passed). +- Review: inspected owned diff; no scope creep or Phase 1 receipt changes. + `PHASE3_SYSTEMS.md` was unrelated untracked work and was not touched. +- Learning: no promotion; implementation-specific fixture layout is documented + locally and no reusable policy change was discovered. +- Verdict: PASS. diff --git a/prototype/README.md b/prototype/README.md index d1f09c3..8e1b35d 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -81,6 +81,32 @@ failed check, and **`2` means the repair ran and the result is not verified**. `2` is not a lesser `1`: it is the case the whole proposal exists to make expressible, so it is a distinct code rather than a warning on stdout. +### Offline semantic conformance + +Semantic probes are separate behavioral evidence, not a change to Phase 1 +receipts. Checked-in synthetic fixtures run without a provider, network, or API +key: + +```bash +python3 -m prototype.cli semantic-test \ + --probes spec/semantic/probes.json \ + --config spec/semantic/verifier-config.json \ + --observations spec/semantic/observations.json \ + --case verified-correction +``` + +The three named cases demonstrate `verified` correction (`0`), failed +supersession (`1`), and unknown erasure (`2`). Output is one canonical JSON +semantic report. Malformed or non-conforming manifests exit `1` without a +traceback. The fixture format and privacy constraints are documented in +[spec/README.md](../spec/README.md#offline-semantic-probe-fixtures). + +The offline runner consumes `RecordedSemanticVerifier`; production adapters +implement `SemanticVerifier` in `semantic.py`. They may evaluate confidential +inputs ephemerally, but only the structured verdict, binding digest, timestamp, +and response digest may enter an observation. Raw output and erased values are +not persisted. + Running it against a real SQLite store produces the result that matters: ```text diff --git a/prototype/cli.py b/prototype/cli.py index ddfc3f4..c2e342e 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -31,6 +31,14 @@ from prototype.lineage import LineageLedger from prototype.receipts import Receipt from prototype.schema import load as load_schema, validate as validate_schema +from prototype.semantic import ( + RecordedSemanticVerifier, + SemanticCoverage, + SemanticObservation, + SemanticProbe, + SemanticProbeRunner, + VerifierConfig, +) from prototype.signing import Ed25519Signer from prototype.sqlite_store import SqliteAdapter from prototype.workspace import Workspace @@ -41,6 +49,61 @@ EXIT_INCONCLUSIVE = 2 +def _load_json(path: Path, *, label: str) -> object: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"{label} is not readable JSON") from error + + +def _probe_case(path: Path, case_name: str) -> tuple[SemanticProbe, ...]: + value = _load_json(path, label="probe manifest") + if not isinstance(value, dict) or set(value) != {"cases"}: + raise ValueError("probe manifest must contain only cases") + cases = value["cases"] + if not isinstance(cases, dict) or not all(isinstance(name, str) for name in cases): + raise ValueError("probe manifest cases must be an object") + records = cases.get(case_name) + if not isinstance(records, list): + raise ValueError(f"probe case is missing: {case_name}") + return tuple(SemanticProbe.from_dict(item) for item in records) + + +def _observations_for_case(path: Path, case_name: str) -> tuple[SemanticObservation, ...]: + value = _load_json(path, label="observations manifest") + if not isinstance(value, dict) or set(value) != {"cases"}: + raise ValueError("observations manifest must contain only cases") + cases = value["cases"] + if not isinstance(cases, dict) or not all(isinstance(name, str) for name in cases): + raise ValueError("observations manifest cases must be an object") + records = cases.get(case_name) + if not isinstance(records, list): + raise ValueError(f"observations case is missing: {case_name}") + return tuple(SemanticObservation.from_dict(item) for item in records) + + +def cmd_semantic_test(ws: Workspace, args: argparse.Namespace) -> int: + """Run checked-in or supplied offline semantic observations.""" + + try: + probes = _probe_case(args.probes, args.case) + config = VerifierConfig.from_dict(_load_json(args.config, label="verifier configuration")) + observations = _observations_for_case(args.observations, args.case) + report = SemanticProbeRunner().run( + probes, config, RecordedSemanticVerifier(observations) + ) + except ValueError as error: + print(f"invalid input: {error}", file=sys.stderr) + return EXIT_REFUSED + + print(report.canonical_json()) + return { + SemanticCoverage.VERIFIED: EXIT_OK, + SemanticCoverage.FAILED: EXIT_REFUSED, + SemanticCoverage.UNKNOWN: EXIT_INCONCLUSIVE, + }[report.coverage] + + def _importer(ws: Workspace) -> tuple[Importer, SqliteAdapter]: ledger = ws.load_lineage() store = SqliteAdapter(ws.store_path, ledger, name="sqlite") @@ -269,6 +332,7 @@ def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: "attest": cmd_attest, "audit": cmd_audit, "verify": cmd_verify, + "semantic-test": cmd_semantic_test, } @@ -318,13 +382,21 @@ def build_parser() -> argparse.ArgumentParser: audit.add_argument("--json", action="store_true") sub.add_parser("verify", help="check every receipt's signature and schema") + + semantic_test = sub.add_parser( + "semantic-test", help="run recorded semantic-probe observations offline" + ) + semantic_test.add_argument("--probes", required=True, type=Path) + semantic_test.add_argument("--config", required=True, type=Path) + semantic_test.add_argument("--observations", required=True, type=Path) + semantic_test.add_argument("--case", required=True) return parser def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) ws = Workspace(args.workspace) - if args.command != "init" and not ws.exists(): + if args.command not in {"init", "semantic-test"} and not ws.exists(): print( f"no workspace at {args.workspace}; run `errata init` first", file=sys.stderr, diff --git a/spec/README.md b/spec/README.md index f3e5635..368f8f0 100644 --- a/spec/README.md +++ b/spec/README.md @@ -9,6 +9,40 @@ than this one can be built and checked against the same contract. | `receipt.schema.json` | What an importer returns after acting on one. | | `vectors/manifest.json` | Every vector, which schema it belongs to, whether it must validate, and — for an invalid vector — the text that must appear in the rejection. | | `vendor/json-schema-test-suite/` | Cases from the official JSON-Schema-Test-Suite, vendored unmodified. | +| `semantic/probes.json` | Named, strict semantic-probe sets for offline conformance. | +| `semantic/verifier-config.json` | Exact synthetic verifier configuration, whose canonical digest binds every observation. | +| `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | + +## Offline semantic-probe fixtures + +The semantic fixtures exercise provider-neutral behavioral evidence without a +network connection, API key, or raw model output. Run one named case with: + +```bash +python3 -m prototype.cli semantic-test \ + --probes spec/semantic/probes.json \ + --config spec/semantic/verifier-config.json \ + --observations spec/semantic/observations.json \ + --case verified-correction +``` + +`probes.json` and `observations.json` are strict objects containing only a +`cases` object. The selected case must exist in both files. A probe case is an +array of strict `SemanticProbe` records. An observation case is an array of +strict `SemanticObservation` records. `verifier-config.json` is one strict +`VerifierConfig` record. Unknown fields, malformed JSON, missing cases, and +invalid record fields are refused rather than interpreted. + +The included synthetic cases are `verified-correction`, +`failed-supersession`, and `unknown-erasure`. The command prints one canonical +JSON `SemanticProbeReport` and exits `0` for `verified`, `1` for `failed` or +invalid input, and `2` for `unknown`. + +Erasure fixtures carry only fixed content-free protocol labels, timestamps, +verdicts, and digests. They never contain a retired value or raw provider +response. A live provider adapter belongs at the `SemanticVerifier` seam in +`prototype/semantic.py`; its confidential prompts and raw output must remain +outside persisted manifests and reports. ## Running the vectors diff --git a/spec/semantic/observations.json b/spec/semantic/observations.json new file mode 100644 index 0000000..9fc518f --- /dev/null +++ b/spec/semantic/observations.json @@ -0,0 +1,74 @@ +{ + "cases": { + "failed-supersession": [ + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:00Z", + "operation": "supersede", + "probe_id": "supersede-negative-v1", + "response_digest": "0707070707070707070707070707070707070707070707070707070707070707", + "verdict": "fail" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:01Z", + "operation": "supersede", + "probe_id": "supersede-positive-v1", + "response_digest": "0808080808080808080808080808080808080808080808080808080808080808", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:02Z", + "operation": "supersede", + "probe_id": "supersede-preservation-v1", + "response_digest": "0909090909090909090909090909090909090909090909090909090909090909", + "verdict": "pass" + } + ], + "unknown-erasure": [ + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:00Z", + "operation": "erase", + "probe_id": "erase-negative-v1", + "response_digest": "0404040404040404040404040404040404040404040404040404040404040404", + "verdict": "inconclusive" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:01Z", + "operation": "erase", + "probe_id": "erase-preservation-v1", + "response_digest": "0505050505050505050505050505050505050505050505050505050505050505", + "verdict": "pass" + } + ], + "verified-correction": [ + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:00Z", + "operation": "correct", + "probe_id": "correct-negative-v1", + "response_digest": "0101010101010101010101010101010101010101010101010101010101010101", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:01Z", + "operation": "correct", + "probe_id": "correct-positive-v1", + "response_digest": "0202020202020202020202020202020202020202020202020202020202020202", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:02Z", + "operation": "correct", + "probe_id": "correct-preservation-v1", + "response_digest": "0303030303030303030303030303030303030303030303030303030303030303", + "verdict": "pass" + } + ] + } +} diff --git a/spec/semantic/probes.json b/spec/semantic/probes.json new file mode 100644 index 0000000..876dc2b --- /dev/null +++ b/spec/semantic/probes.json @@ -0,0 +1,74 @@ +{ + "cases": { + "verified-correction": [ + { + "kind": "negative", + "operation": "correct", + "probe_id": "correct-negative-v1", + "prompt_template": "synthetic-negative-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "positive", + "operation": "correct", + "probe_id": "correct-positive-v1", + "prompt_template": "synthetic-positive-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "preservation", + "operation": "correct", + "probe_id": "correct-preservation-v1", + "prompt_template": "synthetic-preservation-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + } + ], + "failed-supersession": [ + { + "kind": "negative", + "operation": "supersede", + "probe_id": "supersede-negative-v1", + "prompt_template": "synthetic-negative-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "positive", + "operation": "supersede", + "probe_id": "supersede-positive-v1", + "prompt_template": "synthetic-positive-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "preservation", + "operation": "supersede", + "probe_id": "supersede-preservation-v1", + "prompt_template": "synthetic-preservation-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + } + ], + "unknown-erasure": [ + { + "kind": "negative", + "operation": "erase", + "probe_id": "erase-negative-v1", + "prompt_template": "erasure-content-free-v1", + "required": true, + "scope": "declared-store-set-v1" + }, + { + "kind": "preservation", + "operation": "erase", + "probe_id": "erase-preservation-v1", + "prompt_template": "erasure-content-free-v1", + "required": true, + "scope": "declared-store-set-v1" + } + ] + } +} diff --git a/spec/semantic/verifier-config.json b/spec/semantic/verifier-config.json new file mode 100644 index 0000000..1f2c513 --- /dev/null +++ b/spec/semantic/verifier-config.json @@ -0,0 +1,9 @@ +{ + "model": "recorded-semantic-verifier-v1", + "prompt_template_version": "semantic-fixtures-v1", + "provider": "synthetic-provider", + "sampling": { + "temperature": 0 + }, + "seed": 7 +} diff --git a/tests/test_cli.py b/tests/test_cli.py index c5872d5..5368474 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,6 +16,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] +SEMANTIC_FIXTURES = REPO_ROOT / "spec" / "semantic" EXIT_OK = 0 EXIT_REFUSED = 1 @@ -200,5 +201,50 @@ def test_a_second_repair_finds_nothing_to_do(self) -> None: self.assertIn("nothing to repair", result.stdout) +class SemanticProbeConformance(CliCase): + def semantic_test(self, case: str, *extra: str) -> subprocess.CompletedProcess[str]: + return self.run_cli( + "semantic-test", + "--probes", str(SEMANTIC_FIXTURES / "probes.json"), + "--config", str(SEMANTIC_FIXTURES / "verifier-config.json"), + "--observations", str(SEMANTIC_FIXTURES / "observations.json"), + "--case", case, + *extra, + ) + + def test_checked_in_cases_return_coverage_exit_codes_and_canonical_reports(self) -> None: + expected = { + "verified-correction": EXIT_OK, + "failed-supersession": EXIT_REFUSED, + "unknown-erasure": EXIT_INCONCLUSIVE, + } + for case, exit_code in expected.items(): + with self.subTest(case=case): + result = self.semantic_test(case) + self.assertEqual(result.returncode, exit_code, result.stderr) + self.assertEqual(result.stderr, "") + self.assertNotIn("\n", result.stdout.rstrip("\n")) + report = json.loads(result.stdout) + self.assertEqual(report["coverage"], case.split("-", 1)[0]) + self.assertEqual( + result.stdout, + json.dumps(report, sort_keys=True, separators=(",", ":")) + "\n", + ) + + def test_semantic_test_refuses_malformed_input_without_traceback(self) -> None: + bad_observations = self.cwd / "bad-observations.json" + bad_observations.write_text("{not json", encoding="utf-8") + result = self.run_cli( + "semantic-test", + "--probes", str(SEMANTIC_FIXTURES / "probes.json"), + "--config", str(SEMANTIC_FIXTURES / "verifier-config.json"), + "--observations", str(bad_observations), + "--case", "verified-correction", + ) + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertIn("invalid input", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + if __name__ == "__main__": unittest.main() From c73127189e6ccc41e7f85a103fc87ca697759235 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 17:11:15 +0200 Subject: [PATCH 10/88] fix: complete semantic conformance cases --- .../task-2-report.md | 18 +++ prototype/README.md | 15 +- prototype/cli.py | 2 +- spec/README.md | 20 ++- spec/semantic/observations.json | 130 ++++++++++++++++++ spec/semantic/probes.json | 130 ++++++++++++++++++ tests/test_cli.py | 40 ++++-- 7 files changed, 330 insertions(+), 25 deletions(-) diff --git a/.superpowers/sdd/2026-08-12-semantic-probes-and-publication/task-2-report.md b/.superpowers/sdd/2026-08-12-semantic-probes-and-publication/task-2-report.md index 89a0331..825ebfa 100644 --- a/.superpowers/sdd/2026-08-12-semantic-probes-and-publication/task-2-report.md +++ b/.superpowers/sdd/2026-08-12-semantic-probes-and-publication/task-2-report.md @@ -28,3 +28,21 @@ - Learning: no promotion; implementation-specific fixture layout is documented locally and no reusable policy change was discovered. - Verdict: PASS. + +## Review remediation — revision 2 + +- Scope: Task 2 review findings only. +- Change: `--case` defaults to `verified-correction`; exact approved invocation + now succeeds without it. Added named provider-error, missing-response, + duplicate-response, configuration-drift, and nonconforming-output fixtures. +- Evidence gate: `tests.test_cli.SemanticProbeConformance` enumerates all eight + cases, their required exit codes, canonical output, and expected limitation + evidence. Provider error uses structured `error`; nonconforming output uses + a structurally valid operation mismatch. Malformed JSON remains invalid input. +- Gates: PASS — focused CLI/semantic suite (44 passed); PASS — full suite (211 + passed); PASS — `make check`; PASS — remediation diff has no whitespace or + ownership violations. +- Evidence: exact invocation, every named semantic case, required coverage + status, canonical JSON, and expected limitation are asserted in + `SemanticProbeConformance`. +- Verdict: PASS. diff --git a/prototype/README.md b/prototype/README.md index 8e1b35d..2526b6d 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -91,14 +91,17 @@ key: python3 -m prototype.cli semantic-test \ --probes spec/semantic/probes.json \ --config spec/semantic/verifier-config.json \ - --observations spec/semantic/observations.json \ - --case verified-correction + --observations spec/semantic/observations.json ``` -The three named cases demonstrate `verified` correction (`0`), failed -supersession (`1`), and unknown erasure (`2`). Output is one canonical JSON -semantic report. Malformed or non-conforming manifests exit `1` without a -traceback. The fixture format and privacy constraints are documented in +This exact invocation defaults to `verified-correction`. Pass `--case NAME` to +run named cases. The fixtures demonstrate verified correction (`0`), failed +supersession (`1`), and unknown erasure, provider error, missing response, +duplicate response, configuration drift, and structurally parseable +nonconforming output (`2`). Output is one canonical JSON semantic report. +Malformed manifests exit `1` without a traceback; an unexpected structured +verifier record is semantic uncertainty, not an argument-parser error. Fixture +format and privacy constraints are documented in [spec/README.md](../spec/README.md#offline-semantic-probe-fixtures). The offline runner consumes `RecordedSemanticVerifier`; production adapters diff --git a/prototype/cli.py b/prototype/cli.py index c2e342e..8f188a6 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -389,7 +389,7 @@ def build_parser() -> argparse.ArgumentParser: semantic_test.add_argument("--probes", required=True, type=Path) semantic_test.add_argument("--config", required=True, type=Path) semantic_test.add_argument("--observations", required=True, type=Path) - semantic_test.add_argument("--case", required=True) + semantic_test.add_argument("--case", default="verified-correction") return parser diff --git a/spec/README.md b/spec/README.md index 368f8f0..f1d84f1 100644 --- a/spec/README.md +++ b/spec/README.md @@ -16,27 +16,33 @@ than this one can be built and checked against the same contract. ## Offline semantic-probe fixtures The semantic fixtures exercise provider-neutral behavioral evidence without a -network connection, API key, or raw model output. Run one named case with: +network connection, API key, or raw model output. The approved invocation runs +the default `verified-correction` case: ```bash python3 -m prototype.cli semantic-test \ --probes spec/semantic/probes.json \ --config spec/semantic/verifier-config.json \ - --observations spec/semantic/observations.json \ - --case verified-correction + --observations spec/semantic/observations.json ``` `probes.json` and `observations.json` are strict objects containing only a -`cases` object. The selected case must exist in both files. A probe case is an +`cases` object. `--case NAME` selects a non-default case; selected cases must +exist in both files. A probe case is an array of strict `SemanticProbe` records. An observation case is an array of strict `SemanticObservation` records. `verifier-config.json` is one strict `VerifierConfig` record. Unknown fields, malformed JSON, missing cases, and invalid record fields are refused rather than interpreted. The included synthetic cases are `verified-correction`, -`failed-supersession`, and `unknown-erasure`. The command prints one canonical -JSON `SemanticProbeReport` and exits `0` for `verified`, `1` for `failed` or -invalid input, and `2` for `unknown`. +`failed-supersession`, `unknown-erasure`, `provider-error`, +`missing-response`, `duplicate-response`, `configuration-drift`, and +`nonconforming-output`. The latter five prove coverage refuses provider error, +missing evidence, duplicate evidence, binding drift, and a structurally +parseable operation mismatch. They are semantic `unknown` outcomes, unlike +malformed manifest shape, which is invalid CLI input. The command prints one +canonical JSON `SemanticProbeReport` and exits `0` for `verified`, `1` for +`failed` or invalid input, and `2` for `unknown`. Erasure fixtures carry only fixed content-free protocol labels, timestamps, verdicts, and digests. They never contain a retired value or raw provider diff --git a/spec/semantic/observations.json b/spec/semantic/observations.json index 9fc518f..a532389 100644 --- a/spec/semantic/observations.json +++ b/spec/semantic/observations.json @@ -44,6 +44,136 @@ "verdict": "pass" } ], + "provider-error": [ + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:00Z", + "operation": "correct", + "probe_id": "provider-error-negative-v1", + "response_digest": "1010101010101010101010101010101010101010101010101010101010101010", + "verdict": "error" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:01Z", + "operation": "correct", + "probe_id": "provider-error-positive-v1", + "response_digest": "1111111111111111111111111111111111111111111111111111111111111111", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:02Z", + "operation": "correct", + "probe_id": "provider-error-preservation-v1", + "response_digest": "1212121212121212121212121212121212121212121212121212121212121212", + "verdict": "pass" + } + ], + "missing-response": [ + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:00Z", + "operation": "correct", + "probe_id": "missing-negative-v1", + "response_digest": "1313131313131313131313131313131313131313131313131313131313131313", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:02Z", + "operation": "correct", + "probe_id": "missing-preservation-v1", + "response_digest": "1414141414141414141414141414141414141414141414141414141414141414", + "verdict": "pass" + } + ], + "duplicate-response": [ + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:00Z", + "operation": "correct", + "probe_id": "duplicate-negative-v1", + "response_digest": "1515151515151515151515151515151515151515151515151515151515151515", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:01Z", + "operation": "correct", + "probe_id": "duplicate-negative-v1", + "response_digest": "1616161616161616161616161616161616161616161616161616161616161616", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:02Z", + "operation": "correct", + "probe_id": "duplicate-positive-v1", + "response_digest": "1717171717171717171717171717171717171717171717171717171717171717", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:03Z", + "operation": "correct", + "probe_id": "duplicate-preservation-v1", + "response_digest": "1818181818181818181818181818181818181818181818181818181818181818", + "verdict": "pass" + } + ], + "configuration-drift": [ + { + "config_digest": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "observed_at": "2026-08-12T09:00:00Z", + "operation": "correct", + "probe_id": "drift-negative-v1", + "response_digest": "1919191919191919191919191919191919191919191919191919191919191919", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:01Z", + "operation": "correct", + "probe_id": "drift-positive-v1", + "response_digest": "2020202020202020202020202020202020202020202020202020202020202020", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:02Z", + "operation": "correct", + "probe_id": "drift-preservation-v1", + "response_digest": "2121212121212121212121212121212121212121212121212121212121212121", + "verdict": "pass" + } + ], + "nonconforming-output": [ + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:00Z", + "operation": "supersede", + "probe_id": "nonconforming-negative-v1", + "response_digest": "2222222222222222222222222222222222222222222222222222222222222222", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:01Z", + "operation": "correct", + "probe_id": "nonconforming-positive-v1", + "response_digest": "2323232323232323232323232323232323232323232323232323232323232323", + "verdict": "pass" + }, + { + "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", + "observed_at": "2026-08-12T09:00:02Z", + "operation": "correct", + "probe_id": "nonconforming-preservation-v1", + "response_digest": "2424242424242424242424242424242424242424242424242424242424242424", + "verdict": "pass" + } + ], "verified-correction": [ { "config_digest": "cb6f4284ee4d7946c8f970ef2025c9e6d2fd41d553405a2b99ca287ab4faf125", diff --git a/spec/semantic/probes.json b/spec/semantic/probes.json index 876dc2b..1e55cd9 100644 --- a/spec/semantic/probes.json +++ b/spec/semantic/probes.json @@ -69,6 +69,136 @@ "required": true, "scope": "declared-store-set-v1" } + ], + "provider-error": [ + { + "kind": "negative", + "operation": "correct", + "probe_id": "provider-error-negative-v1", + "prompt_template": "synthetic-negative-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "positive", + "operation": "correct", + "probe_id": "provider-error-positive-v1", + "prompt_template": "synthetic-positive-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "preservation", + "operation": "correct", + "probe_id": "provider-error-preservation-v1", + "prompt_template": "synthetic-preservation-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + } + ], + "missing-response": [ + { + "kind": "negative", + "operation": "correct", + "probe_id": "missing-negative-v1", + "prompt_template": "synthetic-negative-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "positive", + "operation": "correct", + "probe_id": "missing-positive-v1", + "prompt_template": "synthetic-positive-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "preservation", + "operation": "correct", + "probe_id": "missing-preservation-v1", + "prompt_template": "synthetic-preservation-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + } + ], + "duplicate-response": [ + { + "kind": "negative", + "operation": "correct", + "probe_id": "duplicate-negative-v1", + "prompt_template": "synthetic-negative-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "positive", + "operation": "correct", + "probe_id": "duplicate-positive-v1", + "prompt_template": "synthetic-positive-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "preservation", + "operation": "correct", + "probe_id": "duplicate-preservation-v1", + "prompt_template": "synthetic-preservation-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + } + ], + "configuration-drift": [ + { + "kind": "negative", + "operation": "correct", + "probe_id": "drift-negative-v1", + "prompt_template": "synthetic-negative-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "positive", + "operation": "correct", + "probe_id": "drift-positive-v1", + "prompt_template": "synthetic-positive-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "preservation", + "operation": "correct", + "probe_id": "drift-preservation-v1", + "prompt_template": "synthetic-preservation-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + } + ], + "nonconforming-output": [ + { + "kind": "negative", + "operation": "correct", + "probe_id": "nonconforming-negative-v1", + "prompt_template": "synthetic-negative-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "positive", + "operation": "correct", + "probe_id": "nonconforming-positive-v1", + "prompt_template": "synthetic-positive-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + }, + { + "kind": "preservation", + "operation": "correct", + "probe_id": "nonconforming-preservation-v1", + "prompt_template": "synthetic-preservation-v1", + "required": true, + "scope": "synthetic-declared-scope-v1" + } ] } } diff --git a/tests/test_cli.py b/tests/test_cli.py index 5368474..ad993b3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -202,35 +202,53 @@ def test_a_second_repair_finds_nothing_to_do(self) -> None: class SemanticProbeConformance(CliCase): - def semantic_test(self, case: str, *extra: str) -> subprocess.CompletedProcess[str]: - return self.run_cli( + def semantic_test( + self, case: str | None = None, *extra: str + ) -> subprocess.CompletedProcess[str]: + arguments = [ "semantic-test", "--probes", str(SEMANTIC_FIXTURES / "probes.json"), "--config", str(SEMANTIC_FIXTURES / "verifier-config.json"), "--observations", str(SEMANTIC_FIXTURES / "observations.json"), - "--case", case, - *extra, - ) + ] + if case is not None: + arguments.extend(("--case", case)) + return self.run_cli(*arguments, *extra) - def test_checked_in_cases_return_coverage_exit_codes_and_canonical_reports(self) -> None: + def test_checked_in_cases_return_coverage_exit_codes_and_limitations(self) -> None: expected = { - "verified-correction": EXIT_OK, - "failed-supersession": EXIT_REFUSED, - "unknown-erasure": EXIT_INCONCLUSIVE, + "verified-correction": (EXIT_OK, "verified", None), + "failed-supersession": (EXIT_REFUSED, "failed", None), + "unknown-erasure": (EXIT_INCONCLUSIVE, "unknown", "inconclusive"), + "provider-error": (EXIT_INCONCLUSIVE, "unknown", "returned error"), + "missing-response": (EXIT_INCONCLUSIVE, "unknown", "missing required observation"), + "duplicate-response": (EXIT_INCONCLUSIVE, "unknown", "duplicate observations"), + "configuration-drift": (EXIT_INCONCLUSIVE, "unknown", "configuration drift"), + "nonconforming-output": (EXIT_INCONCLUSIVE, "unknown", "operation mismatch"), } - for case, exit_code in expected.items(): + for case, (exit_code, coverage, limitation) in expected.items(): with self.subTest(case=case): result = self.semantic_test(case) self.assertEqual(result.returncode, exit_code, result.stderr) self.assertEqual(result.stderr, "") self.assertNotIn("\n", result.stdout.rstrip("\n")) report = json.loads(result.stdout) - self.assertEqual(report["coverage"], case.split("-", 1)[0]) + self.assertEqual(report["coverage"], coverage) + if limitation is not None: + self.assertTrue( + any(limitation in item for item in report["limitations"]), + report["limitations"], + ) self.assertEqual( result.stdout, json.dumps(report, sort_keys=True, separators=(",", ":")) + "\n", ) + def test_exact_documented_invocation_defaults_to_verified_correction(self) -> None: + result = self.semantic_test() + self.assertEqual(result.returncode, EXIT_OK, result.stderr) + self.assertEqual(json.loads(result.stdout)["coverage"], "verified") + def test_semantic_test_refuses_malformed_input_without_traceback(self) -> None: bad_observations = self.cwd / "bad-observations.json" bad_observations.write_text("{not json", encoding="utf-8") From 091a9c134ee10254af0de39213a54c4ebd838193 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 17:19:42 +0200 Subject: [PATCH 11/88] docs: open independent LLM Errata validation program --- CHANGELOG.md | 1 + PHASE3_SYSTEMS.md | 35 +++++++++++++++++++++++++ PRODUCTION_READINESS.md | 4 +-- PUBLISHING.md | 2 ++ README.md | 8 ++++-- ROADMAP.md | 8 +++--- readiness/production-readiness.json | 16 +++++++++--- scripts/validate_repo.py | 40 +++++++++++++++++++++++++++++ tests/test_validate_repo.py | 28 ++++++++++++++++++++ 9 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 PHASE3_SYSTEMS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d4bcb77..dc83b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Added provider-neutral semantic probes, deterministic recorded fixtures, and `errata semantic-test` documentation as Phase 2 item 6 internal implementation. Opened independent review, independent implementation, and Phase 3 system-nomination programs. G2 remains `BLOCKED`: repository artifacts and local tests do not substitute for dated independent external review. - Added a fail-closed production-readiness ledger and human verification matrix. Local green tests cannot produce `PROD_READY`; every required gate must pass, and external gates require dated evidence naming an independent producer. - Added drift checks binding README maturity and SECURITY support policy to `VERSION`. The current verdict remains `NOT_PROD_READY`: only G1, document consistency, is complete. - Normalize complete-cell Markdown presentation decoration before readiness-matrix duplicate and contradiction checks. diff --git a/PHASE3_SYSTEMS.md b/PHASE3_SYSTEMS.md new file mode 100644 index 0000000..19cc323 --- /dev/null +++ b/PHASE3_SYSTEMS.md @@ -0,0 +1,35 @@ +# Phase 3 system nominations + +LLM Errata is seeking nominations for three independently operated AI-memory +systems for a synthetic-data interoperability experiment. + +Repository: https://github.com/thomaswillner/llm-errata + +Current verdict: **NOT_PROD_READY**. The experiment has not started. + +## Required experiment shape + +One synthetic memory root must receive a correction, supersession, and erasure +across three independently operated systems. The set must include: + +- one intentionally nonconforming importer; +- one system whose coverage is genuinely incomplete or opaque; and +- at least one system capable of recording derivation across mixed artifacts. + +Measurements include observation-to-quarantine time, known-descendant coverage, +stale-behavior rate, replacement activation, collateral retention, stale-reimport +resistance, opaque coverage, operator effort, and user-visible friction. + +## Nomination requirements + +Please name: + +- system, operator, public documentation, and version; +- memory stores and derivation surfaces exposed; +- API, account, cost, and data-residency requirements; +- deletion, correction, export, and audit capabilities; +- known opaque stores or unsupported coverage; and +- whether the operator can authorize a synthetic experiment and publish results. + +Nomination is not approval. Every system, account, integration, cost, and +interaction requires explicit owner authorization before the experiment begins. diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index ccc26c3..a69d48f 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Phase 2 item 6 is implemented and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | `ROADMAP.md` records Phase 2 scope; no qualifying independent review is recorded. | Dated independent external conformance-review result covering complete Phase 2 surface. | +| G2 | All Phase 2 items, including semantic probes, are implemented and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | `ROADMAP.md`, `prototype/semantic.py`, `spec/semantic/`, CLI documentation, and review request record internal implementation; no qualifying independent review is recorded. | Dated independent external conformance-review result covering complete Phase 2 surface. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` records current security scope; no qualifying production cryptography evidence is recorded. | Dated independent security review and evidence of audited production signer, rotation, recovery, and delegation. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | @@ -23,7 +23,7 @@ Build health means the repository's deterministic structure/metadata checks, bou ## Approval boundaries -- External review: obtain authorization before requesting, transmitting artifacts for, or recording an external review. +- External review: obtain authorization before requesting, transmitting artifacts for, or recording an external review. [REVIEW_REQUEST.md](REVIEW_REQUEST.md) is a public request template, not external review evidence. - Third-party systems: obtain approval for each system, account, integration, and interaction before interoperability work. - Real data: synthetic data is required unless explicit approval names permitted real-data scope, handling, and retention. - Final verdict change: only an authorized maintainer may change the ledger verdict after every gate has qualifying evidence; implementation, local testing, and agent review alone cannot do so. diff --git a/PUBLISHING.md b/PUBLISHING.md index 3993bbc..239bb5a 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -30,6 +30,7 @@ Do not call the repository a standard, certified protocol, proven deletion syste 13. Rename `.github/CODEOWNERS.example` to `.github/CODEOWNERS` and replace `@GITHUB_HANDLE` with the real account, so step 12 is enforced rather than remembered. 14. Confirm the `validate` workflow has run green on `main` at least once, and make it a required status check for pull requests. 15. Create the `prior-art`, `correction`, `conformance`, `implementation`, and `maintenance` labels used by the issue forms and Dependabot. +16. Publish [REVIEW_REQUEST.md](REVIEW_REQUEST.md), [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md), and [PHASE3_SYSTEMS.md](PHASE3_SYSTEMS.md) only as calls for evidence. Record an external review, independent implementation, or system experiment in the readiness ledger only after its dated, independently produced result exists. ## Suggested first commit @@ -67,3 +68,4 @@ The repository was published on 2026-08-07 at https://github.com/thomaswillner/l - update `RESEARCH.md`, `PRIOR_ART.md`, and `CHANGELOG.md` together when the bounded claim changes; - record unsuccessful implementation experiments, not only successes; - cut a new version whenever published semantics change. +- keep [docs/PUBLICATION_STRATEGY.md](docs/PUBLICATION_STRATEGY.md) aligned with canonical repository URL, `NOT_PROD_READY` status, and external-evidence boundaries; a post, nomination, or invitation never upgrades readiness. diff --git a/README.md b/README.md index 25f981b..ef28477 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,10 @@ See [PRIOR_ART.md](PRIOR_ART.md) for the feature-level comparison and [RESEARCH. | [CITATION.cff](CITATION.cff) | Citation metadata. | | [CHANGELOG.md](CHANGELOG.md) | Public version history, including claims that were narrowed or retired. | | [PUBLISHING.md](PUBLISHING.md) | Exact repository settings, review gates, release text, and announcement wording. | +| [REVIEW_REQUEST.md](REVIEW_REQUEST.md) | Required record for independent conformance and security/distributed-systems review. | +| [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md) | Call and evidence requirements for independently authored adapters. | +| [PHASE3_SYSTEMS.md](PHASE3_SYSTEMS.md) | Nominations for future authorized three-system synthetic-data experiment. | +| [docs/PUBLICATION_STRATEGY.md](docs/PUBLICATION_STRATEGY.md) | Evidence-bounded publication channels and canonical announcement copy. | ## Verifying this repository @@ -162,7 +166,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and Phase 2 items 1 through 5 are implemented; neither has completed external conformance review, and Phase 2 item 6 remains unstarted. +Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and all six Phase 2 items are internally implemented. Item 6 adds provider-neutral semantic probes with deterministic fixtures and recorded verifier configuration; it does not supply independent external evidence. G2 remains `BLOCKED` pending a dated independent review of the complete Phase 2 conformance surface. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. @@ -176,7 +180,7 @@ The most useful contributions are: 4. a smaller design that achieves the same user outcome; 5. a concrete adapter or conformance test. -Please use the evidence requirements in [CONTRIBUTING.md](CONTRIBUTING.md). A convincing prior-art collision should narrow or retire the claim rather than be argued away. +Please use the evidence requirements in [CONTRIBUTING.md](CONTRIBUTING.md), [REVIEW_REQUEST.md](REVIEW_REQUEST.md), and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). A convincing prior-art collision should narrow or retire the claim rather than be argued away. ## Authorship and research disclosure diff --git a/ROADMAP.md b/ROADMAP.md index 006407e..c97a6f1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -102,7 +102,7 @@ Kill or redesign the concept if exact lineage cannot survive even the controlled ## Phase 2 — Conformance surface -**Status:** items 1 through 5 implemented, not yet externally reviewed. `spec/` carries the schemas and vectors, `prototype/cli.py` the control plane, `prototype/sqlite_store.py` a real transactional store, and `prototype/residue.py` the substrate-evidence rule. Item 6, model-assisted semantic probes, is deliberately not started. +**Status:** items 1 through 6 are internally implemented, not yet externally reviewed. `spec/` carries schemas, vectors, and deterministic semantic fixtures; `prototype/cli.py` exposes the control plane and `semantic-test`; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. This internal milestone does not complete G2: a dated independent external conformance review of the complete Phase 2 surface remains required. Only after the file-backed proof passes: @@ -124,13 +124,13 @@ Only after the file-backed proof passes: 4. Define an adapter interface for enumeration, quarantine, reconstruction, verification, and coverage reporting. 5. Publish conformance vectors for signatures, sequencing, key rotation, concurrent events, invalid targets, receipt binding, and confidentiality. -6. Add model-assisted semantic probes only behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. +6. Add model-assisted semantic probes behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. **Implemented internally:** [`prototype/semantic.py`](prototype/semantic.py), [`spec/semantic/`](spec/semantic/), and `errata semantic-test` record only structured, configuration-bound observations; inconclusive, malformed, missing, duplicate, or drifted required evidence is not success. -Phase 2 is successful when two independently implemented adapters can consume the same erratum and produce receipts that a third-party validator evaluates consistently. +Phase 2 internal implementation is complete. G2 remains blocked until a dated independent external reviewer evaluates the complete conformance surface. Broader Phase 2 interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). ## Phase 3 — Interoperability experiment -Run one user-controlled root across at least three independently operated runtimes or memory systems. +Run one user-controlled root across at least three independently operated runtimes or memory systems. [PHASE3_SYSTEMS.md](PHASE3_SYSTEMS.md) records public system-nomination requirements; nomination is not experiment approval. Measure: diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index 9310a6b..526fb46 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -2,7 +2,7 @@ "schema_version": 1, "project_version": "0.3.0", "verdict": "NOT_PROD_READY", - "last_reviewed": "2026-08-10", + "last_reviewed": "2026-08-12", "gates": [ { "id": "G1", @@ -29,8 +29,18 @@ "name": "Phase 2 complete with external conformance review", "class": "external", "status": "BLOCKED", - "criterion": "Phase 2 item 6 is implemented and an independent reviewer evaluates the complete conformance surface.", - "evidence": [{"kind": "repository", "ref": "ROADMAP.md"}] + "criterion": "All Phase 2 items, including provider-neutral semantic probes, are implemented and an independent reviewer evaluates the complete conformance surface.", + "evidence": [ + {"kind": "repository", "ref": "ROADMAP.md"}, + {"kind": "repository", "ref": "prototype/semantic.py"}, + {"kind": "repository", "ref": "spec/semantic/probes.json"}, + {"kind": "repository", "ref": "spec/semantic/verifier-config.json"}, + {"kind": "repository", "ref": "spec/semantic/observations.json"}, + {"kind": "repository", "ref": "prototype/README.md"}, + {"kind": "repository", "ref": "spec/README.md"}, + {"kind": "repository", "ref": "REVIEW_REQUEST.md"}, + {"kind": "repository", "ref": "INDEPENDENT_IMPLEMENTATION.md"} + ] }, { "id": "G3", diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 95de205..43873e5 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -40,6 +40,10 @@ "CITATION.cff", "CHANGELOG.md", "PUBLISHING.md", + "REVIEW_REQUEST.md", + "INDEPENDENT_IMPLEMENTATION.md", + "PHASE3_SYSTEMS.md", + "docs/PUBLICATION_STRATEGY.md", "VERSION", "LICENSE", "NOTICE", @@ -387,6 +391,41 @@ def check_publication_metadata(reporter: Reporter) -> None: ) +def check_g2_independent_review_gate(reporter: Reporter) -> None: + """Keep internal Phase 2 work from being represented as external review.""" + + path = ROOT / "readiness" / "production-readiness.json" + if not path.is_file(): + return + try: + import json + + payload = json.loads(read_utf8(path)) + gates = payload.get("gates") if isinstance(payload, dict) else None + g2 = next( + (gate for gate in gates if isinstance(gate, dict) and gate.get("id") == "G2"), + None, + ) if isinstance(gates, list) else None + status = g2.get("status") if isinstance(g2, dict) else None + evidence = g2.get("evidence") if isinstance(g2, dict) else None + qualifying_external_review = any( + isinstance(entry, dict) + and entry.get("kind") == "external" + and isinstance(entry.get("ref"), str) + and isinstance(entry.get("producer"), str) + and isinstance(entry.get("observed"), str) + for entry in evidence + ) if isinstance(evidence, list) else False + valid = status != "PASS" or qualifying_external_review + except (ValueError, OSError, UnicodeError): + valid = False + reporter.check( + "G2 independent review gate", + valid, + "G2 PASS is backed by dated, independently produced external review evidence", + "Keep G2 BLOCKED until its evidence includes dated external review by an independent producer.", + ) + def check_document_version_alignment( reporter: Reporter, repository_version: str | None ) -> None: @@ -546,6 +585,7 @@ def main() -> int: check_local_paths(reporter) check_local_links(reporter) check_publication_metadata(reporter) + check_g2_independent_review_gate(reporter) check_document_version_alignment(reporter, repository_version) check_citation(reporter, repository_version) return reporter.finish() diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index d672c74..fb423cb 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -8,6 +8,7 @@ from __future__ import annotations import unittest +import json from pathlib import Path from tests.support import EXIT_FAIL, EXIT_OK, check_after, repo_copy, rewrite, run_checker @@ -40,6 +41,33 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("required files", result.stdout) + def test_missing_independent_validation_program_artifacts_are_rejected(self) -> None: + for relative_path in ( + "REVIEW_REQUEST.md", + "INDEPENDENT_IMPLEMENTATION.md", + "PHASE3_SYSTEMS.md", + "docs/PUBLICATION_STRATEGY.md", + ): + with self.subTest(relative_path=relative_path): + def mutate(root: Path, path: str = relative_path) -> None: + (root / path).unlink() + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("required files", result.stdout) + + def test_internal_phase_two_completion_cannot_upgrade_g2(self) -> None: + def mutate(root: Path) -> None: + path = root / "readiness" / "production-readiness.json" + payload = json.loads(path.read_text(encoding="utf-8")) + gate = next(gate for gate in payload["gates"] if gate["id"] == "G2") + gate["status"] = "PASS" + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("G2 independent review gate", result.stdout) + def test_broken_repository_relative_link_is_rejected(self) -> None: def mutate(root: Path) -> None: rewrite(root / "README.md", "(PRIOR_ART.md)", "(PRIOR_ARTS.md)") From c7d29bf65026cc3ee97a1f90a4a961b8e4c60cfd Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 17:26:34 +0200 Subject: [PATCH 12/88] fix: validate external G2 evidence semantics --- PRODUCTION_READINESS.md | 2 +- scripts/check_readiness.py | 62 +++++++++++++++++++++++++++++++++++-- scripts/validate_repo.py | 6 ++-- tests/test_readiness.py | 33 ++++++++++++++++++++ tests/test_validate_repo.py | 22 +++++++++++++ 5 files changed, 118 insertions(+), 7 deletions(-) diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index a69d48f..7b84098 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | All Phase 2 items, including semantic probes, are implemented and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | `ROADMAP.md`, `prototype/semantic.py`, `spec/semantic/`, CLI documentation, and review request record internal implementation; no qualifying independent review is recorded. | Dated independent external conformance-review result covering complete Phase 2 surface. | +| G2 | All Phase 2 items, including provider-neutral semantic probes, are implemented and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Internal Phase 2 implementation is recorded in repository evidence; no qualifying independent review is recorded. | Dated independent external conformance-review result covering complete Phase 2 surface. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` records current security scope; no qualifying production cryptography evidence is recorded. | Dated independent security review and evidence of audited production signer, rotation, recovery, and delegation. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 7440779..bbe299f 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -29,6 +29,14 @@ r"(?:^|[\s:_-])(local|self|maintainer|agent|repository|repo)(?:$|[\s:_-])", re.IGNORECASE, ) +G2_MATRIX_CURRENT_EVIDENCE = ( + "Internal Phase 2 implementation is recorded in repository evidence; " + "no qualifying independent review is recorded." +) +G2_MATRIX_NEXT_EVIDENCE = ( + "Dated independent external conformance-review result covering complete " + "Phase 2 surface." +) class Reporter: @@ -79,6 +87,24 @@ def valid_external_ref(value: object) -> bool: return value.startswith("urn:") or (parsed.scheme == "https" and bool(parsed.netloc)) +def valid_external_evidence(entry: object, *, today: date | None = None) -> bool: + """Return whether one external evidence entry meets readiness semantics.""" + + if not isinstance(entry, dict): + return False + producer = entry.get("producer") + observed = entry.get("observed") + if not valid_external_ref(entry.get("ref")): + return False + if not isinstance(producer, str) or not producer.strip(): + return False + if NON_INDEPENDENT_PRODUCER_RE.search(producer) is not None: + return False + if not valid_iso_date(observed): + return False + return date.fromisoformat(observed) <= (today or date.today()) + + def markdown_row_cells(line: str) -> list[str] | None: if not line.startswith("|") or not line.endswith("|"): return None @@ -191,6 +217,33 @@ def validate_matrix( "all rows must be unique, well formed, and exact", ) + g2_gate = next( + (gate for gate in raw_gates if isinstance(gate, dict) and gate.get("id") == "G2"), + None, + ) if isinstance(raw_gates, list) else None + g2_row = next( + (row for row in gate_rows if len(row) == 5 and markdown_value(row[0]) == "G2"), + None, + ) + g2_criterion = g2_gate.get("criterion") if isinstance(g2_gate, dict) else None + reporter.check( + "G2 matrix criterion", + isinstance(g2_criterion, str) + and g2_row is not None + and markdown_value(g2_row[1]) == g2_criterion, + "G2 matrix criterion exactly matches the readiness ledger", + ) + reporter.check( + "G2 matrix current evidence", + g2_row is not None and markdown_value(g2_row[3]) == G2_MATRIX_CURRENT_EVIDENCE, + "G2 matrix uses the canonical internal-evidence statement", + ) + reporter.check( + "G2 matrix next evidence", + g2_row is not None and markdown_value(g2_row[4]) == G2_MATRIX_NEXT_EVIDENCE, + "G2 matrix requires dated independent external review", + ) + def validate_ledger( payload: dict[str, object], repository_version: str, reporter: Reporter @@ -325,7 +378,10 @@ def validate_ledger( and bool(producer.strip()) and NON_INDEPENDENT_PRODUCER_RE.search(producer) is None ) - observed_valid = valid_iso_date(entry.get("observed")) + observed_valid = ( + valid_iso_date(entry.get("observed")) + and date.fromisoformat(entry["observed"]) <= date.today() + ) reporter.check( f"{entry_name} external reference", reference_valid, @@ -339,9 +395,9 @@ def validate_ledger( reporter.check( f"{entry_name} observed date", observed_valid, - "observed must be an ISO YYYY-MM-DD date", + "observed must be an ISO YYYY-MM-DD date that is not future-dated", ) - entry_valid = reference_valid and producer_valid and observed_valid + entry_valid = valid_external_evidence(entry) evidence_valid = evidence_valid and entry_valid if entry_valid: valid_external_entries += 1 diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 43873e5..d001d47 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -21,6 +21,8 @@ from typing import Iterable from urllib.parse import unquote +from check_readiness import valid_external_evidence + ROOT = Path(__file__).resolve().parents[1] @@ -411,9 +413,7 @@ def check_g2_independent_review_gate(reporter: Reporter) -> None: qualifying_external_review = any( isinstance(entry, dict) and entry.get("kind") == "external" - and isinstance(entry.get("ref"), str) - and isinstance(entry.get("producer"), str) - and isinstance(entry.get("observed"), str) + and valid_external_evidence(entry) for entry in evidence ) if isinstance(evidence, list) else False valid = status != "PASS" or qualifying_external_review diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 14367a9..701df96 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -170,6 +170,39 @@ def mutate(root): result = check_after(SCRIPT, mutate) self.assert_rejected_without_traceback(result, "matrix gate statuses") + def test_g2_matrix_criterion_drift_is_rejected(self) -> None: + def mutate(root): + rewrite( + root / "PRODUCTION_READINESS.md", + "All Phase 2 items, including provider-neutral semantic probes, are implemented and an independent reviewer evaluates the complete conformance surface.", + "Internal Phase 2 implementation is enough.", + ) + + result = check_after(SCRIPT, mutate) + self.assert_rejected_without_traceback(result, "G2 matrix criterion") + + def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: + def mutate(root): + rewrite( + root / "PRODUCTION_READINESS.md", + "Internal Phase 2 implementation is recorded in repository evidence; no qualifying independent review is recorded.", + "local tests prove readiness.", + ) + + result = check_after(SCRIPT, mutate) + self.assert_rejected_without_traceback(result, "G2 matrix current evidence") + + def test_g2_matrix_next_evidence_drift_is_rejected(self) -> None: + def mutate(root): + rewrite( + root / "PRODUCTION_READINESS.md", + "Dated independent external conformance-review result covering complete Phase 2 surface.", + "Local tests are enough.", + ) + + result = check_after(SCRIPT, mutate) + self.assert_rejected_without_traceback(result, "G2 matrix next evidence") + def test_code_formatted_duplicate_matrix_gate_is_rejected(self) -> None: def mutate(root): path = root / "PRODUCTION_READINESS.md" diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index fb423cb..f41276f 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -68,6 +68,28 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("G2 independent review gate", result.stdout) + def test_g2_pass_rejects_malformed_external_review_evidence(self) -> None: + invalid_reviews = ( + {"ref": "", "producer": "Independent reviewer", "observed": "2026-08-09"}, + {"ref": "ftp://example.invalid/review", "producer": "Independent reviewer", "observed": "2026-08-09"}, + {"ref": "urn:example:review", "producer": "local implementer", "observed": "2026-08-09"}, + {"ref": "urn:example:review", "producer": "Independent reviewer", "observed": "invalid"}, + {"ref": "urn:example:review", "producer": "Independent reviewer", "observed": "2999-01-01"}, + ) + for review in invalid_reviews: + with self.subTest(review=review): + def mutate(root: Path, evidence: dict[str, str] = review) -> None: + path = root / "readiness" / "production-readiness.json" + payload = json.loads(path.read_text(encoding="utf-8")) + gate = next(gate for gate in payload["gates"] if gate["id"] == "G2") + gate["status"] = "PASS" + gate["evidence"].append({"kind": "external", **evidence}) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("G2 independent review gate", result.stdout) + def test_broken_repository_relative_link_is_rejected(self) -> None: def mutate(root: Path) -> None: rewrite(root / "README.md", "(PRIOR_ART.md)", "(PRIOR_ARTS.md)") From 4d9ee11574513a2b5453c2fc46d2771c21b49550 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 17:32:49 +0200 Subject: [PATCH 13/88] fix: require complete independent G2 review evidence --- REVIEW_REQUEST.md | 31 ++++++++++++++++++- scripts/check_readiness.py | 60 ++++++++++++++++++++++++++++++++++--- scripts/validate_repo.py | 4 +-- tests/test_readiness.py | 43 +++++++++++++++++++++++++- tests/test_validate_repo.py | 51 +++++++++++++++++++++++++++++++ 5 files changed, 181 insertions(+), 8 deletions(-) diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md index 8815fb5..913c3fb 100644 --- a/REVIEW_REQUEST.md +++ b/REVIEW_REQUEST.md @@ -41,7 +41,36 @@ Please identify: - conflicts of interest or prior involvement; and - verdict: proceed, narrow, redesign, or retire. +## G2 ledger schema + +A review can qualify G2 only when its public external report is represented in +the readiness ledger with all fields below. A `fail` result is recorded as +evidence but cannot make G2 pass. + +```json +{ + "kind": "external", + "ref": "https://reviewer.example/report", + "producer": "Independent reviewer identity", + "observed": "YYYY-MM-DD", + "review_type": "phase2-conformance", + "reviewed_commit": "40 lowercase hexadecimal characters", + "scope": [ + "schemas", "vectors", "cli", "adapter-interface", + "transactional-store", "substrate-evidence", "semantic-probes", + "security-boundaries" + ], + "result": "pass | pass-with-findings | fail", + "relationship": "independent-third-party", + "conflicts": [] +} +``` + +`ref` must be a public `https` URL with host and path, or a well-formed +`urn::`. `observed` cannot be future-dated. `conflicts` is always an +array, including when empty. A qualifying pass requires `pass` or +`pass-with-findings` plus every listed scope token. + Open a GitHub issue for public findings. Security-sensitive findings must follow `SECURITY.md`. Review invitations and automated reviews are not independent evidence for the production-readiness ledger. - diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index bbe299f..2cbfcb3 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -26,9 +26,23 @@ STATUSES = {"PASS", "FAIL", "BLOCKED"} CLASSES = {"internal", "external"} NON_INDEPENDENT_PRODUCER_RE = re.compile( - r"(?:^|[\s:_-])(local|self|maintainer|agent|repository|repo)(?:$|[\s:_-])", + r"(?:^|[\s:_-])(local|self|maintainer|agent|repository|repo|project[\s_-]*owner|reference[\s_-]*implementer)(?:$|[\s:_-])", re.IGNORECASE, ) +URN_RE = re.compile(r"^urn:[A-Za-z0-9][A-Za-z0-9-]{1,31}:[^\s]+$") +G2_SCOPE = frozenset( + { + "schemas", + "vectors", + "cli", + "adapter-interface", + "transactional-store", + "substrate-evidence", + "semantic-probes", + "security-boundaries", + } +) +G2_RESULTS = {"pass", "pass-with-findings", "fail"} G2_MATRIX_CURRENT_EVIDENCE = ( "Internal Phase 2 implementation is recorded in repository evidence; " "no qualifying independent review is recorded." @@ -84,7 +98,9 @@ def valid_external_ref(value: object) -> bool: parsed = urlparse(value) except ValueError: return False - return value.startswith("urn:") or (parsed.scheme == "https" and bool(parsed.netloc)) + return bool(URN_RE.fullmatch(value)) or ( + parsed.scheme == "https" and bool(parsed.netloc) and bool(parsed.path) + ) def valid_external_evidence(entry: object, *, today: date | None = None) -> bool: @@ -105,6 +121,35 @@ def valid_external_evidence(entry: object, *, today: date | None = None) -> bool return date.fromisoformat(observed) <= (today or date.today()) +def valid_g2_review_evidence(entry: object, *, today: date | None = None) -> bool: + """Validate a qualifying independent review of complete Phase 2 surface.""" + + if not valid_external_evidence(entry, today=today) or not isinstance(entry, dict): + return False + scope = entry.get("scope") + return ( + entry.get("kind") == "external" + and entry.get("review_type") == "phase2-conformance" + and isinstance(entry.get("reviewed_commit"), str) + and re.fullmatch(r"[0-9a-f]{40}", entry["reviewed_commit"]) is not None + and isinstance(scope, list) + and all(isinstance(token, str) for token in scope) + and G2_SCOPE.issubset(scope) + and entry.get("result") in G2_RESULTS + and entry.get("relationship") == "independent-third-party" + and isinstance(entry.get("conflicts"), list) + ) + + +def qualifying_g2_review_evidence(entry: object, *, today: date | None = None) -> bool: + """Return whether a valid G2 review can satisfy a G2 PASS gate.""" + + return valid_g2_review_evidence(entry, today=today) and entry.get("result") in { + "pass", + "pass-with-findings", + } + + def markdown_row_cells(line: str) -> list[str] | None: if not line.startswith("|") or not line.endswith("|"): return None @@ -339,6 +384,7 @@ def validate_ledger( all_pass = False valid_external_entries = 0 + valid_g2_reviews = 0 evidence_valid = isinstance(evidence, list) if isinstance(evidence, list): for index, entry in enumerate(evidence): @@ -401,6 +447,8 @@ def validate_ledger( evidence_valid = evidence_valid and entry_valid if entry_valid: valid_external_entries += 1 + if gate_id == "G2" and qualifying_g2_review_evidence(entry): + valid_g2_reviews += 1 else: reporter.check(entry_name, False, "kind must be repository or external") evidence_valid = False @@ -408,11 +456,15 @@ def validate_ledger( if not evidence_valid: all_pass = False - external_pass_valid = gate_class != "external" or status != "PASS" or valid_external_entries > 0 + external_pass_valid = ( + gate_class != "external" + or status != "PASS" + or (valid_g2_reviews > 0 if gate_id == "G2" else valid_external_entries > 0) + ) reporter.check( f"{prefix} external PASS evidence", external_pass_valid, - "external PASS requires independently observed external evidence", + "G2 PASS requires a complete independent Phase 2 review; other external PASS gates require independently observed external evidence", ) if not external_pass_valid: all_pass = False diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index d001d47..cd32888 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -21,7 +21,7 @@ from typing import Iterable from urllib.parse import unquote -from check_readiness import valid_external_evidence +from check_readiness import qualifying_g2_review_evidence ROOT = Path(__file__).resolve().parents[1] @@ -413,7 +413,7 @@ def check_g2_independent_review_gate(reporter: Reporter) -> None: qualifying_external_review = any( isinstance(entry, dict) and entry.get("kind") == "external" - and valid_external_evidence(entry) + and qualifying_g2_review_evidence(entry) for entry in evidence ) if isinstance(evidence, list) else False valid = status != "PASS" or qualifying_external_review diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 701df96..a66a03f 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -5,7 +5,7 @@ import json import unittest -from scripts.check_readiness import markdown_value +from scripts.check_readiness import qualifying_g2_review_evidence, markdown_value from tests.support import ( EXIT_FAIL, EXIT_OK, @@ -25,6 +25,25 @@ def test_current_not_ready_ledger_is_honest(self) -> None: result = run_checker(root, SCRIPT) self.assertEqual(result.returncode, EXIT_OK, result.stdout) + def test_complete_independent_g2_review_passes_schema(self) -> None: + review = { + "kind": "external", + "ref": "https://reviews.example.org/phase2/report", + "producer": "Independent Systems Lab", + "observed": "2026-08-12", + "review_type": "phase2-conformance", + "reviewed_commit": "a" * 40, + "scope": [ + "schemas", "vectors", "cli", "adapter-interface", + "transactional-store", "substrate-evidence", "semantic-probes", + "security-boundaries", + ], + "result": "pass-with-findings", + "relationship": "independent-third-party", + "conflicts": [], + } + self.assertTrue(qualifying_g2_review_evidence(review)) + def test_consistently_formatted_canonical_matrix_cells_are_accepted(self) -> None: def mutate(root): path = root / "PRODUCTION_READINESS.md" @@ -328,6 +347,28 @@ def mutate(payload): self.assertEqual(result.returncode, EXIT_FAIL) self.assertIn("independence", result.stdout) + def test_g2_pass_requires_complete_independent_review_schema(self) -> None: + def mutate(payload): + gate = next(gate for gate in payload["gates"] if gate["id"] == "G2") + gate["status"] = "PASS" + gate["evidence"].append( + { + "kind": "external", + "ref": "https://reviews.example.org/phase2/report", + "producer": "Independent Systems Lab", + "observed": "2026-08-12", + "review_type": "phase2-conformance", + "reviewed_commit": "a" * 40, + "scope": ["schemas"], + "result": "pass-with-findings", + "relationship": "independent-third-party", + "conflicts": [], + } + ) + + result = self._mutated(mutate) + self.assert_rejected_without_traceback(result, "G2 external PASS evidence") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index f41276f..1b8abfc 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -90,6 +90,57 @@ def mutate(root: Path, evidence: dict[str, str] = review) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("G2 independent review gate", result.stdout) + def test_g2_pass_requires_complete_independent_review_schema(self) -> None: + review = { + "kind": "external", + "ref": "https://reviews.example.org/phase2/report", + "producer": "Independent Systems Lab", + "observed": "2026-08-12", + "review_type": "phase2-conformance", + "reviewed_commit": "a" * 40, + "scope": [ + "schemas", "vectors", "cli", "adapter-interface", + "transactional-store", "substrate-evidence", "semantic-probes", + "security-boundaries", + ], + "result": "pass-with-findings", + "relationship": "independent-third-party", + "conflicts": [], + } + invalid_reviews = [] + for field in ("kind", "review_type", "reviewed_commit", "scope", "result", "relationship", "conflicts"): + invalid = dict(review) + invalid.pop(field) + invalid_reviews.append(invalid) + for field, value in ( + ("kind", "repository"), + ("ref", "urn:"), + ("producer", "project owner"), + ("producer", "reference implementer"), + ("reviewed_commit", "a" * 39), + ("scope", ["schemas"]), + ("result", "fail"), + ("relationship", "maintainer"), + ("conflicts", "none"), + ): + invalid = dict(review) + invalid[field] = value + invalid_reviews.append(invalid) + + for evidence in invalid_reviews: + with self.subTest(evidence=evidence): + def mutate(root: Path, review_entry: dict[str, object] = evidence) -> None: + path = root / "readiness" / "production-readiness.json" + payload = json.loads(path.read_text(encoding="utf-8")) + gate = next(gate for gate in payload["gates"] if gate["id"] == "G2") + gate["status"] = "PASS" + gate["evidence"].append(review_entry) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("G2 independent review gate", result.stdout) + def test_broken_repository_relative_link_is_rejected(self) -> None: def mutate(root: Path) -> None: rewrite(root / "README.md", "(PRIOR_ART.md)", "(PRIOR_ARTS.md)") From 941396204685a631ac9bb095bc1868938c1f6828 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 17:39:45 +0200 Subject: [PATCH 14/88] fix: bind G2 reviews to current conformance surface --- REVIEW_REQUEST.md | 36 +++++++++++-- scripts/check_readiness.py | 105 +++++++++++++++++++++++++++++++----- scripts/validate_repo.py | 16 ++++-- tests/test_readiness.py | 38 ++++++++++++- tests/test_validate_repo.py | 16 +++++- 5 files changed, 186 insertions(+), 25 deletions(-) diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md index 913c3fb..31009fa 100644 --- a/REVIEW_REQUEST.md +++ b/REVIEW_REQUEST.md @@ -62,14 +62,42 @@ evidence but cannot make G2 pass. ], "result": "pass | pass-with-findings | fail", "relationship": "independent-third-party", - "conflicts": [] + "conflicts": [], + "producer_identity": "https://identity.example/reviewer", + "independence_attestation": "llm-errata-independent-review-v1", + "surface_digest": "SHA-256 of canonical Phase 2 surface" } ``` `ref` must be a public `https` URL with host and path, or a well-formed -`urn::`. `observed` cannot be future-dated. `conflicts` is always an -array, including when empty. A qualifying pass requires `pass` or -`pass-with-findings` plus every listed scope token. +`urn::`. `producer_identity` must be an `https` URL with host and +path. `observed` cannot be future-dated. `conflicts` is always an array, +including when empty. Scope must contain each token exactly once. A qualifying +pass requires `pass` or `pass-with-findings` plus every listed scope token. + +`surface_digest` is SHA-256 over each file below in listed order, appending the +UTF-8 repository-relative path, one NUL byte, then the file's raw bytes: + +```text +spec/erratum.schema.json +spec/receipt.schema.json +spec/vectors/manifest.json +prototype/cli.py +prototype/adapters.py +prototype/sqlite_store.py +prototype/residue.py +prototype/semantic.py +spec/semantic/probes.json +spec/semantic/verifier-config.json +spec/semantic/observations.json +SECURITY.md +THREAT_MODEL.md +``` + +The checker validates schema, claimed relationship, report identity, commit +binding, and surface digest. It cannot prove the reviewer is independent or +that conflicts are complete; a human must verify those claims before changing +the readiness verdict. Open a GitHub issue for public findings. Security-sensitive findings must follow `SECURITY.md`. Review invitations and automated reviews are not independent diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 2cbfcb3..875ce1e 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -5,6 +5,8 @@ import json import re +import hashlib +import subprocess import sys from datetime import date from pathlib import Path @@ -26,10 +28,26 @@ STATUSES = {"PASS", "FAIL", "BLOCKED"} CLASSES = {"internal", "external"} NON_INDEPENDENT_PRODUCER_RE = re.compile( - r"(?:^|[\s:_-])(local|self|maintainer|agent|repository|repo|project[\s_-]*owner|reference[\s_-]*implementer)(?:$|[\s:_-])", + r"(?:^|[\s:_-])(author|owner|implementer|contributor|maintainer|thomas|willner|project|reference|local|agent|repository|repo|self)(?:$|[\s:_-])", re.IGNORECASE, ) URN_RE = re.compile(r"^urn:[A-Za-z0-9][A-Za-z0-9-]{1,31}:[^\s]+$") +G2_ATTESTATION = "llm-errata-independent-review-v1" +G2_SURFACE_FILES = ( + "spec/erratum.schema.json", + "spec/receipt.schema.json", + "spec/vectors/manifest.json", + "prototype/cli.py", + "prototype/adapters.py", + "prototype/sqlite_store.py", + "prototype/residue.py", + "prototype/semantic.py", + "spec/semantic/probes.json", + "spec/semantic/verifier-config.json", + "spec/semantic/observations.json", + "SECURITY.md", + "THREAT_MODEL.md", +) G2_SCOPE = frozenset( { "schemas", @@ -92,6 +110,16 @@ def valid_iso_date(value: object) -> bool: def valid_external_ref(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlparse(value) + except ValueError: + return False + return bool(URN_RE.fullmatch(value)) or (parsed.scheme == "https" and bool(parsed.netloc)) + + +def valid_g2_report_ref(value: object) -> bool: if not isinstance(value, str): return False try: @@ -99,8 +127,44 @@ def valid_external_ref(value: object) -> bool: except ValueError: return False return bool(URN_RE.fullmatch(value)) or ( - parsed.scheme == "https" and bool(parsed.netloc) and bool(parsed.path) + parsed.scheme == "https" and bool(parsed.netloc) and parsed.path not in {"", "/"} + ) + + +def valid_g2_identity_ref(value: object) -> bool: + if not isinstance(value, str): + return False + try: + parsed = urlparse(value) + except ValueError: + return False + return parsed.scheme == "https" and bool(parsed.netloc) and parsed.path not in {"", "/"} + + +def g2_surface_digest(root: Path = ROOT) -> str: + """SHA-256 of sorted `path + NUL + bytes` canonical Phase 2 surface.""" + + digest = hashlib.sha256() + for relative in G2_SURFACE_FILES: + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update((root / relative).read_bytes()) + return digest.hexdigest() + + +def reviewed_commit_exists(value: str, root: Path = ROOT) -> bool: + """Check commit existence when this checkout has live Git metadata.""" + + if not (root / ".git").exists(): + return True + result = subprocess.run( + ["git", "cat-file", "-e", f"{value}^{{commit}}"], + cwd=root, + capture_output=True, + check=False, + text=True, ) + return result.returncode == 0 def valid_external_evidence(entry: object, *, today: date | None = None) -> bool: @@ -127,18 +191,29 @@ def valid_g2_review_evidence(entry: object, *, today: date | None = None) -> boo if not valid_external_evidence(entry, today=today) or not isinstance(entry, dict): return False scope = entry.get("scope") - return ( - entry.get("kind") == "external" - and entry.get("review_type") == "phase2-conformance" - and isinstance(entry.get("reviewed_commit"), str) - and re.fullmatch(r"[0-9a-f]{40}", entry["reviewed_commit"]) is not None - and isinstance(scope, list) - and all(isinstance(token, str) for token in scope) - and G2_SCOPE.issubset(scope) - and entry.get("result") in G2_RESULTS - and entry.get("relationship") == "independent-third-party" - and isinstance(entry.get("conflicts"), list) - ) + reviewed_commit = entry.get("reviewed_commit") + try: + return ( + entry.get("kind") == "external" + and valid_g2_report_ref(entry.get("ref")) + and entry.get("review_type") == "phase2-conformance" + and isinstance(reviewed_commit, str) + and re.fullmatch(r"[0-9a-f]{40}", reviewed_commit) is not None + and reviewed_commit_exists(reviewed_commit) + and isinstance(scope, list) + and all(isinstance(token, str) for token in scope) + and len(scope) == len(set(scope)) + and set(scope) == G2_SCOPE + and entry.get("result") in G2_RESULTS + and entry.get("relationship") == "independent-third-party" + and isinstance(entry.get("conflicts"), list) + and isinstance(entry.get("producer_identity"), str) + and valid_g2_identity_ref(entry["producer_identity"]) + and entry.get("independence_attestation") == G2_ATTESTATION + and entry.get("surface_digest") == g2_surface_digest() + ) + except OSError: + return False def qualifying_g2_review_evidence(entry: object, *, today: date | None = None) -> bool: @@ -444,6 +519,8 @@ def validate_ledger( "observed must be an ISO YYYY-MM-DD date that is not future-dated", ) entry_valid = valid_external_evidence(entry) + if gate_id == "G2": + entry_valid = valid_g2_review_evidence(entry) evidence_valid = evidence_valid and entry_valid if entry_valid: valid_external_entries += 1 diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index cd32888..cc1ec42 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -21,7 +21,7 @@ from typing import Iterable from urllib.parse import unquote -from check_readiness import qualifying_g2_review_evidence +from check_readiness import qualifying_g2_review_evidence, valid_g2_review_evidence ROOT = Path(__file__).resolve().parents[1] @@ -416,14 +416,22 @@ def check_g2_independent_review_gate(reporter: Reporter) -> None: and qualifying_g2_review_evidence(entry) for entry in evidence ) if isinstance(evidence, list) else False - valid = status != "PASS" or qualifying_external_review + g2_external_entries_valid = all( + not isinstance(entry, dict) + or entry.get("kind") != "external" + or valid_g2_review_evidence(entry) + for entry in evidence + ) if isinstance(evidence, list) else False + valid = g2_external_entries_valid and ( + status != "PASS" or qualifying_external_review + ) except (ValueError, OSError, UnicodeError): valid = False reporter.check( "G2 independent review gate", valid, - "G2 PASS is backed by dated, independently produced external review evidence", - "Keep G2 BLOCKED until its evidence includes dated external review by an independent producer.", + "G2 external entries are schema-valid declared-independent reviews; a PASS has a qualifying complete review", + "Keep G2 BLOCKED until its evidence includes a complete schema-valid declared-independent review.", ) def check_document_version_alignment( diff --git a/tests/test_readiness.py b/tests/test_readiness.py index a66a03f..596cb78 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -3,9 +3,15 @@ from __future__ import annotations import json +import subprocess import unittest -from scripts.check_readiness import qualifying_g2_review_evidence, markdown_value +from scripts.check_readiness import ( + g2_surface_digest, + qualifying_g2_review_evidence, + valid_external_evidence, + markdown_value, +) from tests.support import ( EXIT_FAIL, EXIT_OK, @@ -26,13 +32,16 @@ def test_current_not_ready_ledger_is_honest(self) -> None: self.assertEqual(result.returncode, EXIT_OK, result.stdout) def test_complete_independent_g2_review_passes_schema(self) -> None: + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], capture_output=True, check=True, text=True + ).stdout.strip() review = { "kind": "external", "ref": "https://reviews.example.org/phase2/report", "producer": "Independent Systems Lab", "observed": "2026-08-12", "review_type": "phase2-conformance", - "reviewed_commit": "a" * 40, + "reviewed_commit": commit, "scope": [ "schemas", "vectors", "cli", "adapter-interface", "transactional-store", "substrate-evidence", "semantic-probes", @@ -41,9 +50,23 @@ def test_complete_independent_g2_review_passes_schema(self) -> None: "result": "pass-with-findings", "relationship": "independent-third-party", "conflicts": [], + "producer_identity": "https://identity.example.org/reviewer", + "independence_attestation": "llm-errata-independent-review-v1", + "surface_digest": g2_surface_digest(), } self.assertTrue(qualifying_g2_review_evidence(review)) + def test_generic_external_evidence_allows_https_root_url(self) -> None: + self.assertTrue( + valid_external_evidence( + { + "ref": "https://reviewer.example.org", + "producer": "Independent Systems Lab", + "observed": "2026-08-12", + } + ) + ) + def test_consistently_formatted_canonical_matrix_cells_are_accepted(self) -> None: def mutate(root): path = root / "PRODUCTION_READINESS.md" @@ -363,12 +386,23 @@ def mutate(payload): "result": "pass-with-findings", "relationship": "independent-third-party", "conflicts": [], + "producer_identity": "https://identity.example.org/reviewer", + "independence_attestation": "llm-errata-independent-review-v1", + "surface_digest": g2_surface_digest(), } ) result = self._mutated(mutate) self.assert_rejected_without_traceback(result, "G2 external PASS evidence") + def test_malformed_g2_external_evidence_is_rejected_while_blocked(self) -> None: + def mutate(payload): + gate = next(gate for gate in payload["gates"] if gate["id"] == "G2") + gate["evidence"].append({"kind": "external", "ref": "urn:"}) + + result = self._mutated(mutate) + self.assert_rejected_without_traceback(result, "G2 evidence") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 1b8abfc..a97189e 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -91,6 +91,8 @@ def mutate(root: Path, evidence: dict[str, str] = review) -> None: self.assertIn("G2 independent review gate", result.stdout) def test_g2_pass_requires_complete_independent_review_schema(self) -> None: + from scripts.check_readiness import g2_surface_digest + review = { "kind": "external", "ref": "https://reviews.example.org/phase2/report", @@ -106,9 +108,15 @@ def test_g2_pass_requires_complete_independent_review_schema(self) -> None: "result": "pass-with-findings", "relationship": "independent-third-party", "conflicts": [], + "producer_identity": "https://identity.example.org/reviewer", + "independence_attestation": "llm-errata-independent-review-v1", + "surface_digest": g2_surface_digest(), } invalid_reviews = [] - for field in ("kind", "review_type", "reviewed_commit", "scope", "result", "relationship", "conflicts"): + for field in ( + "kind", "review_type", "reviewed_commit", "scope", "result", "relationship", + "conflicts", "producer_identity", "independence_attestation", "surface_digest", + ): invalid = dict(review) invalid.pop(field) invalid_reviews.append(invalid) @@ -119,9 +127,15 @@ def test_g2_pass_requires_complete_independent_review_schema(self) -> None: ("producer", "reference implementer"), ("reviewed_commit", "a" * 39), ("scope", ["schemas"]), + ("scope", review["scope"] + ["schemas"]), + ("scope", review["scope"] + ["extra"]), ("result", "fail"), ("relationship", "maintainer"), ("conflicts", "none"), + ("producer_identity", "https://identity.example.org"), + ("producer_identity", "https://identity.example.org/"), + ("independence_attestation", "independent"), + ("surface_digest", "a" * 64), ): invalid = dict(review) invalid[field] = value From c0619c807f7c8c672590f77b13878a9adf9811e4 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 17:46:28 +0200 Subject: [PATCH 15/88] fix: bind G2 review digest to reviewed commit --- REVIEW_REQUEST.md | 48 ++++++++++------ scripts/check_readiness.py | 109 +++++++++++++++++++++++++++---------- scripts/validate_repo.py | 4 +- tests/test_readiness.py | 69 +++++++++++++++++++++++ 4 files changed, 180 insertions(+), 50 deletions(-) diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md index 31009fa..972e994 100644 --- a/REVIEW_REQUEST.md +++ b/REVIEW_REQUEST.md @@ -75,29 +75,41 @@ path. `observed` cannot be future-dated. `conflicts` is always an array, including when empty. Scope must contain each token exactly once. A qualifying pass requires `pass` or `pass-with-findings` plus every listed scope token. -`surface_digest` is SHA-256 over each file below in listed order, appending the -UTF-8 repository-relative path, one NUL byte, then the file's raw bytes: +`surface_digest` is SHA-256 over sorted canonical paths. For every path, append +its UTF-8 repository-relative path, one NUL byte, raw bytes, then one final NUL +byte. Canonical path enumeration is: ```text -spec/erratum.schema.json -spec/receipt.schema.json -spec/vectors/manifest.json -prototype/cli.py -prototype/adapters.py -prototype/sqlite_store.py -prototype/residue.py -prototype/semantic.py -spec/semantic/probes.json -spec/semantic/verifier-config.json -spec/semantic/observations.json -SECURITY.md +all first-party prototype/*.py +prototype/README.md +spec/README.md +all first-party spec/*.schema.json +all first-party spec/vectors/*.json +all first-party spec/semantic/*.json +ROADMAP.md THREAT_MODEL.md +SECURITY.md +tests/test_adapters.py +tests/test_cli.py +tests/test_controller.py +tests/test_ed25519.py +tests/test_errata_feed.py +tests/test_schema.py +tests/test_semantic.py +tests/test_sqlite_store.py ``` -The checker validates schema, claimed relationship, report identity, commit -binding, and surface digest. It cannot prove the reviewer is independent or -that conflicts are complete; a human must verify those claims before changing -the readiness verdict. +Vendor files under `spec/vendor/` are excluded. All named groups must be +nonempty. Use `python3 -c 'from scripts.check_readiness import +g2_surface_digest; print(g2_surface_digest())'` from repository root to print +current digest. + +The checker requires live Git metadata, verifies `reviewed_commit` is a commit, +reads each canonical file at that commit with `git show`, and requires supplied +digest to match both commit and current worktree. It validates schema, claimed +relationship, and report identity. It cannot prove reviewer independence or +complete conflict disclosure; a human must verify those claims before changing +readiness verdict. Open a GitHub issue for public findings. Security-sensitive findings must follow `SECURITY.md`. Review invitations and automated reviews are not independent diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 875ce1e..72ff573 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -27,26 +27,25 @@ VERDICTS = {"NOT_PROD_READY", "PROD_READY"} STATUSES = {"PASS", "FAIL", "BLOCKED"} CLASSES = {"internal", "external"} -NON_INDEPENDENT_PRODUCER_RE = re.compile( +GENERIC_NON_INDEPENDENT_PRODUCER_RE = re.compile( + r"(?:^|[\s:_-])(local|self|maintainer|agent|repository|repo)(?:$|[\s:_-])", + re.IGNORECASE, +) +G2_NON_INDEPENDENT_PRODUCER_RE = re.compile( r"(?:^|[\s:_-])(author|owner|implementer|contributor|maintainer|thomas|willner|project|reference|local|agent|repository|repo|self)(?:$|[\s:_-])", re.IGNORECASE, ) URN_RE = re.compile(r"^urn:[A-Za-z0-9][A-Za-z0-9-]{1,31}:[^\s]+$") G2_ATTESTATION = "llm-errata-independent-review-v1" -G2_SURFACE_FILES = ( - "spec/erratum.schema.json", - "spec/receipt.schema.json", - "spec/vectors/manifest.json", - "prototype/cli.py", - "prototype/adapters.py", - "prototype/sqlite_store.py", - "prototype/residue.py", - "prototype/semantic.py", - "spec/semantic/probes.json", - "spec/semantic/verifier-config.json", - "spec/semantic/observations.json", - "SECURITY.md", - "THREAT_MODEL.md", +G2_REQUIRED_TESTS = ( + "tests/test_adapters.py", + "tests/test_cli.py", + "tests/test_controller.py", + "tests/test_ed25519.py", + "tests/test_errata_feed.py", + "tests/test_schema.py", + "tests/test_semantic.py", + "tests/test_sqlite_store.py", ) G2_SCOPE = frozenset( { @@ -141,22 +140,63 @@ def valid_g2_identity_ref(value: object) -> bool: return parsed.scheme == "https" and bool(parsed.netloc) and parsed.path not in {"", "/"} -def g2_surface_digest(root: Path = ROOT) -> str: - """SHA-256 of sorted `path + NUL + bytes` canonical Phase 2 surface.""" +def g2_surface_files(root: Path = ROOT) -> tuple[str, ...]: + """Return comprehensive sorted first-party Phase 2 review manifest.""" + + groups = ( + tuple(sorted((root / "prototype").glob("*.py"))), + (root / "prototype" / "README.md",), + (root / "spec" / "README.md",), + tuple(sorted((root / "spec").glob("*.schema.json"))), + tuple(sorted((root / "spec" / "vectors").glob("*.json"))), + tuple(sorted((root / "spec" / "semantic").glob("*.json"))), + tuple(root / path for path in ("ROADMAP.md", "THREAT_MODEL.md", "SECURITY.md")), + tuple(root / path for path in G2_REQUIRED_TESTS), + ) + if any(not group for group in groups) or any(not path.is_file() for group in groups for path in group): + raise OSError("canonical G2 surface is incomplete") + return tuple( + sorted(path.relative_to(root).as_posix() for group in groups for path in group) + ) + + +def surface_digest_from_bytes(entries: list[tuple[str, bytes]]) -> str: + """SHA-256 over `relative path + NUL + raw bytes + NUL` ordered entries.""" digest = hashlib.sha256() - for relative in G2_SURFACE_FILES: + for relative, content in entries: digest.update(relative.encode("utf-8")) digest.update(b"\0") - digest.update((root / relative).read_bytes()) + digest.update(content) + digest.update(b"\0") return digest.hexdigest() +def g2_surface_digest(root: Path = ROOT) -> str: + return surface_digest_from_bytes( + [(relative, (root / relative).read_bytes()) for relative in g2_surface_files(root)] + ) + + +def g2_surface_digest_at_commit(commit: str, root: Path = ROOT) -> str: + if not reviewed_commit_exists(commit, root): + raise OSError("reviewed commit is unavailable") + entries = [] + for relative in g2_surface_files(root): + result = subprocess.run( + ["git", "show", f"{commit}:{relative}"], cwd=root, capture_output=True, check=False + ) + if result.returncode != 0: + raise OSError(f"reviewed commit lacks {relative}") + entries.append((relative, result.stdout)) + return surface_digest_from_bytes(entries) + + def reviewed_commit_exists(value: str, root: Path = ROOT) -> bool: - """Check commit existence when this checkout has live Git metadata.""" + """Fail closed unless checkout has Git metadata and commit exists.""" if not (root / ".git").exists(): - return True + return False result = subprocess.run( ["git", "cat-file", "-e", f"{value}^{{commit}}"], cwd=root, @@ -178,14 +218,16 @@ def valid_external_evidence(entry: object, *, today: date | None = None) -> bool return False if not isinstance(producer, str) or not producer.strip(): return False - if NON_INDEPENDENT_PRODUCER_RE.search(producer) is not None: + if GENERIC_NON_INDEPENDENT_PRODUCER_RE.search(producer) is not None: return False if not valid_iso_date(observed): return False return date.fromisoformat(observed) <= (today or date.today()) -def valid_g2_review_evidence(entry: object, *, today: date | None = None) -> bool: +def valid_g2_review_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: """Validate a qualifying independent review of complete Phase 2 surface.""" if not valid_external_evidence(entry, today=today) or not isinstance(entry, dict): @@ -199,7 +241,7 @@ def valid_g2_review_evidence(entry: object, *, today: date | None = None) -> boo and entry.get("review_type") == "phase2-conformance" and isinstance(reviewed_commit, str) and re.fullmatch(r"[0-9a-f]{40}", reviewed_commit) is not None - and reviewed_commit_exists(reviewed_commit) + and G2_NON_INDEPENDENT_PRODUCER_RE.search(entry["producer"]) is None and isinstance(scope, list) and all(isinstance(token, str) for token in scope) and len(scope) == len(set(scope)) @@ -210,16 +252,19 @@ def valid_g2_review_evidence(entry: object, *, today: date | None = None) -> boo and isinstance(entry.get("producer_identity"), str) and valid_g2_identity_ref(entry["producer_identity"]) and entry.get("independence_attestation") == G2_ATTESTATION - and entry.get("surface_digest") == g2_surface_digest() + and entry.get("surface_digest") == g2_surface_digest(root) + and entry.get("surface_digest") == g2_surface_digest_at_commit(reviewed_commit, root) ) except OSError: return False -def qualifying_g2_review_evidence(entry: object, *, today: date | None = None) -> bool: +def qualifying_g2_review_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: """Return whether a valid G2 review can satisfy a G2 PASS gate.""" - return valid_g2_review_evidence(entry, today=today) and entry.get("result") in { + return valid_g2_review_evidence(entry, today=today, root=root) and entry.get("result") in { "pass", "pass-with-findings", } @@ -497,12 +542,16 @@ def validate_ledger( producer_valid = ( isinstance(producer, str) and bool(producer.strip()) - and NON_INDEPENDENT_PRODUCER_RE.search(producer) is None + and GENERIC_NON_INDEPENDENT_PRODUCER_RE.search(producer) is None ) observed_valid = ( valid_iso_date(entry.get("observed")) and date.fromisoformat(entry["observed"]) <= date.today() ) + if gate_id == "G2" and isinstance(producer, str): + producer_valid = producer_valid and ( + G2_NON_INDEPENDENT_PRODUCER_RE.search(producer) is None + ) reporter.check( f"{entry_name} external reference", reference_valid, @@ -520,11 +569,11 @@ def validate_ledger( ) entry_valid = valid_external_evidence(entry) if gate_id == "G2": - entry_valid = valid_g2_review_evidence(entry) + entry_valid = valid_g2_review_evidence(entry, root=ROOT) evidence_valid = evidence_valid and entry_valid if entry_valid: valid_external_entries += 1 - if gate_id == "G2" and qualifying_g2_review_evidence(entry): + if gate_id == "G2" and qualifying_g2_review_evidence(entry, root=ROOT): valid_g2_reviews += 1 else: reporter.check(entry_name, False, "kind must be repository or external") diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index cc1ec42..61f21f0 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -413,13 +413,13 @@ def check_g2_independent_review_gate(reporter: Reporter) -> None: qualifying_external_review = any( isinstance(entry, dict) and entry.get("kind") == "external" - and qualifying_g2_review_evidence(entry) + and qualifying_g2_review_evidence(entry, root=ROOT) for entry in evidence ) if isinstance(evidence, list) else False g2_external_entries_valid = all( not isinstance(entry, dict) or entry.get("kind") != "external" - or valid_g2_review_evidence(entry) + or valid_g2_review_evidence(entry, root=ROOT) for entry in evidence ) if isinstance(evidence, list) else False valid = g2_external_entries_valid and ( diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 596cb78..b8497a8 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -3,13 +3,18 @@ from __future__ import annotations import json +import shutil import subprocess +import tempfile import unittest +from pathlib import Path from scripts.check_readiness import ( g2_surface_digest, + g2_surface_digest_at_commit, qualifying_g2_review_evidence, valid_external_evidence, + valid_g2_review_evidence, markdown_value, ) from tests.support import ( @@ -67,6 +72,70 @@ def test_generic_external_evidence_allows_https_root_url(self) -> None: ) ) + def test_g2_review_binds_current_committed_surface(self) -> None: + with repo_copy() as source, tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "repository" + shutil.copytree(source, root) + for command in ( + ("git", "init"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Readiness tests"), + ("git", "add", "."), + ("git", "commit", "-m", "surface baseline"), + ): + subprocess.run(command, cwd=root, check=True, capture_output=True) + commit = subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + review = self._complete_review(root, commit) + self.assertTrue(valid_g2_review_evidence(review, root=root)) + controller = root / "prototype" / "controller.py" + controller.write_bytes(controller.read_bytes() + b"\n# digest mutation\n") + self.assertNotEqual(g2_surface_digest(root), review["surface_digest"]) + self.assertFalse(valid_g2_review_evidence(review, root=root)) + + def test_g2_review_requires_git_metadata(self) -> None: + with repo_copy() as root: + review = self._complete_review(root, "a" * 40) + self.assertFalse(valid_g2_review_evidence(review, root=root)) + + def test_g2_review_rejects_nonexistent_commit(self) -> None: + with repo_copy() as source, tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "repository" + shutil.copytree(source, root) + for command in ( + ("git", "init"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Readiness tests"), + ("git", "add", "."), + ("git", "commit", "-m", "surface baseline"), + ): + subprocess.run(command, cwd=root, check=True, capture_output=True) + self.assertFalse(valid_g2_review_evidence(self._complete_review(root, "a" * 40), root=root)) + + @staticmethod + def _complete_review(root: Path, commit: str) -> dict[str, object]: + return { + "kind": "external", + "ref": "https://reviews.example.org/phase2/report", + "producer": "Independent Systems Lab", + "observed": "2026-08-12", + "review_type": "phase2-conformance", + "reviewed_commit": commit, + "scope": [ + "schemas", "vectors", "cli", "adapter-interface", + "transactional-store", "substrate-evidence", "semantic-probes", + "security-boundaries", + ], + "result": "pass-with-findings", + "relationship": "independent-third-party", + "conflicts": [], + "producer_identity": "https://identity.example.org/reviewer", + "independence_attestation": "llm-errata-independent-review-v1", + "surface_digest": g2_surface_digest(root), + } + def test_consistently_formatted_canonical_matrix_cells_are_accepted(self) -> None: def mutate(root): path = root / "PRODUCTION_READINESS.md" From 59e739bd846643145993aeccded8f493bad01336 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:00:29 +0200 Subject: [PATCH 16/88] fix: align Phase 2 evidence and link validation --- CHANGELOG.md | 2 +- PRODUCTION_READINESS.md | 2 +- README.md | 2 +- ROADMAP.md | 4 ++-- readiness/production-readiness.json | 2 +- scripts/check_links.py | 22 ++++++++++++++++---- scripts/check_readiness.py | 9 +++++---- tests/test_check_links.py | 24 ++++++++++++++++++++++ tests/test_readiness.py | 31 +++-------------------------- 9 files changed, 56 insertions(+), 42 deletions(-) create mode 100644 tests/test_check_links.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dc83b6a..32c91c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,7 +90,7 @@ All six were found by adversarial review after the suite was green, and each has - `CHANGELOG.md` used `[version]` heading syntax with no link definitions, so the headings rendered as literal brackets, and it listed a placeholder as though it were a change. Version headings are now plain until the repository has a public URL to compare against. -## [0.1.0](https://github.com/thomaswillner/llm-errata/releases/tag/v0.1.0) - 2026-08-01 +## 0.1.0 - 2026-08-01 ### Added diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 7b84098..6571e46 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | All Phase 2 items, including provider-neutral semantic probes, are implemented and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Internal Phase 2 implementation is recorded in repository evidence; no qualifying independent review is recorded. | Dated independent external conformance-review result covering complete Phase 2 surface. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, receipt binding, and confidentiality are absent; no qualifying independent review is recorded. | Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` records current security scope; no qualifying production cryptography evidence is recorded. | Dated independent security review and evidence of audited production signer, rotation, recovery, and delegation. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/README.md b/README.md index ef28477..1be908f 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and all six Phase 2 items are internally implemented. Item 6 adds provider-neutral semantic probes with deterministic fixtures and recorded verifier configuration; it does not supply independent external evidence. G2 remains `BLOCKED` pending a dated independent review of the complete Phase 2 conformance surface. +Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 is implemented; Phase 2 remains incomplete. Item 6 adds provider-neutral semantic probes with deterministic fixtures and recorded verifier configuration, while explicit `errata quarantine` CLI support and vectors for key rotation, concurrent events, invalid targets, receipt binding, and confidentiality remain open. G2 remains `BLOCKED` pending internal Phase 2 completion and a dated independent review of the complete conformance surface. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/ROADMAP.md b/ROADMAP.md index c97a6f1..520377f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -102,7 +102,7 @@ Kill or redesign the concept if exact lineage cannot survive even the controlled ## Phase 2 — Conformance surface -**Status:** items 1 through 6 are internally implemented, not yet externally reviewed. `spec/` carries schemas, vectors, and deterministic semantic fixtures; `prototype/cli.py` exposes the control plane and `semantic-test`; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. This internal milestone does not complete G2: a dated independent external conformance review of the complete Phase 2 surface remains required. +**Status:** items 1 through 5 are partially implemented and item 6 is internally implemented; Phase 2 is not complete. `spec/` carries schemas, vectors, and deterministic semantic fixtures; `prototype/cli.py` exposes most control-plane commands and `semantic-test`; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. The explicit `errata quarantine` command and vectors for key rotation, concurrent events, invalid targets, receipt binding, and confidentiality remain unimplemented. This internal work does not complete G2: a dated independent external conformance review of the complete Phase 2 surface remains required. Only after the file-backed proof passes: @@ -126,7 +126,7 @@ Only after the file-backed proof passes: 5. Publish conformance vectors for signatures, sequencing, key rotation, concurrent events, invalid targets, receipt binding, and confidentiality. 6. Add model-assisted semantic probes behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. **Implemented internally:** [`prototype/semantic.py`](prototype/semantic.py), [`spec/semantic/`](spec/semantic/), and `errata semantic-test` record only structured, configuration-bound observations; inconclusive, malformed, missing, duplicate, or drifted required evidence is not success. -Phase 2 internal implementation is complete. G2 remains blocked until a dated independent external reviewer evaluates the complete conformance surface. Broader Phase 2 interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). +Phase 2 internal implementation is incomplete. Complete the listed CLI and vector gaps before asking an independent reviewer to evaluate the complete conformance surface. G2 remains blocked until that implementation work and a dated independent external review exist. Broader Phase 2 interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). ## Phase 3 — Interoperability experiment diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index 526fb46..261205b 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -29,7 +29,7 @@ "name": "Phase 2 complete with external conformance review", "class": "external", "status": "BLOCKED", - "criterion": "All Phase 2 items, including provider-neutral semantic probes, are implemented and an independent reviewer evaluates the complete conformance surface.", + "criterion": "Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface.", "evidence": [ {"kind": "repository", "ref": "ROADMAP.md"}, {"kind": "repository", "ref": "prototype/semantic.py"}, diff --git a/scripts/check_links.py b/scripts/check_links.py index fca0b35..94728a7 100644 --- a/scripts/check_links.py +++ b/scripts/check_links.py @@ -40,7 +40,22 @@ # Status codes that prove the resource exists but the client was filtered. BLOCKED_STATUSES = frozenset({401, 403, 405, 406, 429, 503}) -URL_PATTERN = re.compile(r"https?://[^\s)>\"'\]]+") +URL_PATTERN = re.compile(r"https?://[^\s)>\"'\]`]+") + + +def is_reserved_example_url(url: str) -> bool: + """Reserved IANA example domains are schema placeholders, not citations.""" + + host = urllib.parse.urlparse(url).hostname or "" + return host == "example" or host.endswith(".example") + + +def extract_urls(text: str) -> set[str]: + return { + match.group(0).rstrip(".,;:") + for match in URL_PATTERN.finditer(text) + if not is_reserved_example_url(match.group(0).rstrip(".,;:")) + } def cited_urls() -> dict[str, list[str]]: @@ -48,11 +63,10 @@ def cited_urls() -> dict[str, list[str]]: found: dict[str, set[str]] = {} for path in sorted(ROOT.rglob("*.md")): - if ".git" in path.parts: + if ".git" in path.parts or ".superpowers" in path.parts or "superpowers" in path.parts: continue text = path.read_text(encoding="utf-8") - for match in URL_PATTERN.finditer(text): - url = match.group(0).rstrip(".,;:") + for url in extract_urls(text): found.setdefault(url, set()).add(path.relative_to(ROOT).as_posix()) return {url: sorted(sources) for url, sources in sorted(found.items())} diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 72ff573..19ac1de 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -61,12 +61,13 @@ ) G2_RESULTS = {"pass", "pass-with-findings", "fail"} G2_MATRIX_CURRENT_EVIDENCE = ( - "Internal Phase 2 implementation is recorded in repository evidence; " - "no qualifying independent review is recorded." + "Semantic probes are internally implemented, but Phase 2 remains incomplete: " + "`errata quarantine` and vectors for key rotation, concurrency, invalid targets, " + "receipt binding, and confidentiality are absent; no qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( - "Dated independent external conformance-review result covering complete " - "Phase 2 surface." + "Complete listed Phase 2 gaps, then record dated independent external " + "conformance-review result covering complete Phase 2 surface." ) diff --git a/tests/test_check_links.py b/tests/test_check_links.py new file mode 100644 index 0000000..490e729 --- /dev/null +++ b/tests/test_check_links.py @@ -0,0 +1,24 @@ +"""Focused extraction tests for external citation liveness checks.""" + +from __future__ import annotations + +import unittest + +from scripts.check_links import extract_urls, is_reserved_example_url + + +class LinkExtractionTests(unittest.TestCase): + def test_markdown_closing_backticks_are_not_part_of_url(self) -> None: + self.assertEqual( + extract_urls("Repository: `https://github.com/thomaswillner/llm-errata`") , + {"https://github.com/thomaswillner/llm-errata"}, + ) + + def test_reserved_example_urls_are_not_live_citations(self) -> None: + self.assertTrue(is_reserved_example_url("https://reviewer.example/report")) + self.assertTrue(is_reserved_example_url("https://identity.example/reviewer")) + self.assertFalse(is_reserved_example_url("https://github.com/openai")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_readiness.py b/tests/test_readiness.py index b8497a8..82464d9 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -36,31 +36,6 @@ def test_current_not_ready_ledger_is_honest(self) -> None: result = run_checker(root, SCRIPT) self.assertEqual(result.returncode, EXIT_OK, result.stdout) - def test_complete_independent_g2_review_passes_schema(self) -> None: - commit = subprocess.run( - ["git", "rev-parse", "HEAD"], capture_output=True, check=True, text=True - ).stdout.strip() - review = { - "kind": "external", - "ref": "https://reviews.example.org/phase2/report", - "producer": "Independent Systems Lab", - "observed": "2026-08-12", - "review_type": "phase2-conformance", - "reviewed_commit": commit, - "scope": [ - "schemas", "vectors", "cli", "adapter-interface", - "transactional-store", "substrate-evidence", "semantic-probes", - "security-boundaries", - ], - "result": "pass-with-findings", - "relationship": "independent-third-party", - "conflicts": [], - "producer_identity": "https://identity.example.org/reviewer", - "independence_attestation": "llm-errata-independent-review-v1", - "surface_digest": g2_surface_digest(), - } - self.assertTrue(qualifying_g2_review_evidence(review)) - def test_generic_external_evidence_allows_https_root_url(self) -> None: self.assertTrue( valid_external_evidence( @@ -285,7 +260,7 @@ def test_g2_matrix_criterion_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "All Phase 2 items, including provider-neutral semantic probes, are implemented and an independent reviewer evaluates the complete conformance surface.", + "Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface.", "Internal Phase 2 implementation is enough.", ) @@ -296,7 +271,7 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Internal Phase 2 implementation is recorded in repository evidence; no qualifying independent review is recorded.", + "Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, receipt binding, and confidentiality are absent; no qualifying independent review is recorded.", "local tests prove readiness.", ) @@ -307,7 +282,7 @@ def test_g2_matrix_next_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Dated independent external conformance-review result covering complete Phase 2 surface.", + "Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface.", "Local tests are enough.", ) From 936240d98397c268c9ff1a64ee9538421550df64 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:03:26 +0200 Subject: [PATCH 17/88] style: remove trailing blank lines --- INDEPENDENT_IMPLEMENTATION.md | 1 - docs/PUBLICATION_STRATEGY.md | 1 - .../plans/2026-08-12-semantic-probes-and-publication.md | 1 - docs/superpowers/specs/2026-08-12-semantic-probes-design.md | 1 - .../specs/2026-08-12-validation-publication-program-design.md | 1 - 5 files changed, 5 deletions(-) diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index f2232b0..8587ebd 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -36,4 +36,3 @@ The current personal-use licence requires written permission to implement the specification. Open a GitHub issue naming the research or standards scope, repository, organization, requested version, and intended publication. A permission grant does not imply technical endorsement. - diff --git a/docs/PUBLICATION_STRATEGY.md b/docs/PUBLICATION_STRATEGY.md index c7a77d9..47a73e7 100644 --- a/docs/PUBLICATION_STRATEGY.md +++ b/docs/PUBLICATION_STRATEGY.md @@ -69,4 +69,3 @@ specialists to review the exact four-part conjunction and identify collisions. For every successful post, record channel, URL, publication timestamp, account, source commit, exact copy digest, and moderation state. A submitted or queued post is not recorded as publicly available until its URL is accessible. - diff --git a/docs/superpowers/plans/2026-08-12-semantic-probes-and-publication.md b/docs/superpowers/plans/2026-08-12-semantic-probes-and-publication.md index fe911ec..da730a9 100644 --- a/docs/superpowers/plans/2026-08-12-semantic-probes-and-publication.md +++ b/docs/superpowers/plans/2026-08-12-semantic-probes-and-publication.md @@ -92,4 +92,3 @@ - [ ] Push `agent/g2-publication` and create a draft PR targeting `agent/prod-readiness` until PR #3 merges. - [ ] Create GitHub issues for independent review, implementation permission/recruitment, and Phase 3 system nominations. - [ ] Publish authenticated external announcements where platform access and rules permit, then record exact URLs without upgrading readiness gates. - diff --git a/docs/superpowers/specs/2026-08-12-semantic-probes-design.md b/docs/superpowers/specs/2026-08-12-semantic-probes-design.md index e155149..bd9dfb3 100644 --- a/docs/superpowers/specs/2026-08-12-semantic-probes-design.md +++ b/docs/superpowers/specs/2026-08-12-semantic-probes-design.md @@ -89,4 +89,3 @@ must continue to run without network access or API keys. Implementation completes Phase 2 item 6 internally. G2 remains `BLOCKED` until an independent reviewer evaluates the schemas, vectors, CLI, adapters, cryptographic boundaries, and semantic-probe layer as one conformance surface. - diff --git a/docs/superpowers/specs/2026-08-12-validation-publication-program-design.md b/docs/superpowers/specs/2026-08-12-validation-publication-program-design.md index f8685a9..df4f07d 100644 --- a/docs/superpowers/specs/2026-08-12-validation-publication-program-design.md +++ b/docs/superpowers/specs/2026-08-12-validation-publication-program-design.md @@ -110,4 +110,3 @@ prohibit the submission, credentials are unavailable, a reviewer is not independent, or implementation requires an unapproved licence change. Reroute after a complete prior-art collision, external review rejection, material G2 contract change, or failure to recruit independent implementers. - From 35052868cde8b6f16fc6c20b4136ae0277f9a00e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:08:41 +0200 Subject: [PATCH 18/88] docs: correct Phase 2 receipt-binding evidence --- PRODUCTION_READINESS.md | 2 +- README.md | 2 +- ROADMAP.md | 2 +- scripts/check_readiness.py | 2 +- tests/test_readiness.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 6571e46..5d2b181 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, receipt binding, and confidentiality are absent; no qualifying independent review is recorded. | Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded. | Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` records current security scope; no qualifying production cryptography evidence is recorded. | Dated independent security review and evidence of audited production signer, rotation, recovery, and delegation. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/README.md b/README.md index 1be908f..16c8cb9 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 is implemented; Phase 2 remains incomplete. Item 6 adds provider-neutral semantic probes with deterministic fixtures and recorded verifier configuration, while explicit `errata quarantine` CLI support and vectors for key rotation, concurrent events, invalid targets, receipt binding, and confidentiality remain open. G2 remains `BLOCKED` pending internal Phase 2 completion and a dated independent review of the complete conformance surface. +Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 is implemented; Phase 2 remains incomplete. Item 6 adds provider-neutral semantic probes with deterministic fixtures and recorded verifier configuration, while explicit `errata quarantine` CLI support and vectors for key rotation, concurrent events, invalid targets, and confidentiality remain open. Receipt-binding vectors exist for state roots but coverage remains partial. G2 remains `BLOCKED` pending internal Phase 2 completion and a dated independent review of the complete conformance surface. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/ROADMAP.md b/ROADMAP.md index 520377f..976c87b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -102,7 +102,7 @@ Kill or redesign the concept if exact lineage cannot survive even the controlled ## Phase 2 — Conformance surface -**Status:** items 1 through 5 are partially implemented and item 6 is internally implemented; Phase 2 is not complete. `spec/` carries schemas, vectors, and deterministic semantic fixtures; `prototype/cli.py` exposes most control-plane commands and `semantic-test`; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. The explicit `errata quarantine` command and vectors for key rotation, concurrent events, invalid targets, receipt binding, and confidentiality remain unimplemented. This internal work does not complete G2: a dated independent external conformance review of the complete Phase 2 surface remains required. +**Status:** items 1 through 5 are partially implemented and item 6 is internally implemented; Phase 2 is not complete. `spec/` carries schemas, vectors, and deterministic semantic fixtures; `prototype/cli.py` exposes most control-plane commands and `semantic-test`; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. The explicit `errata quarantine` command and vectors for key rotation, concurrent events, invalid targets, and confidentiality remain unimplemented. Receipt state-root binding vectors exist, but receipt-binding coverage remains partial. This internal work does not complete G2: a dated independent external conformance review of the complete Phase 2 surface remains required. Only after the file-backed proof passes: diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 19ac1de..cd9ed2f 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -63,7 +63,7 @@ G2_MATRIX_CURRENT_EVIDENCE = ( "Semantic probes are internally implemented, but Phase 2 remains incomplete: " "`errata quarantine` and vectors for key rotation, concurrency, invalid targets, " - "receipt binding, and confidentiality are absent; no qualifying independent review is recorded." + "and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( "Complete listed Phase 2 gaps, then record dated independent external " diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 82464d9..87126af 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -271,7 +271,7 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, receipt binding, and confidentiality are absent; no qualifying independent review is recorded.", + "Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded.", "local tests prove readiness.", ) From cbcfd2a0d77f1060651bac781599edc469a42c26 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:15:04 +0200 Subject: [PATCH 19/88] docs: record public validation calls --- README.md | 1 + docs/PUBLICATION_LOG.md | 47 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 docs/PUBLICATION_LOG.md diff --git a/README.md b/README.md index 16c8cb9..38cdaf8 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ See [PRIOR_ART.md](PRIOR_ART.md) for the feature-level comparison and [RESEARCH. | [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md) | Call and evidence requirements for independently authored adapters. | | [PHASE3_SYSTEMS.md](PHASE3_SYSTEMS.md) | Nominations for future authorized three-system synthetic-data experiment. | | [docs/PUBLICATION_STRATEGY.md](docs/PUBLICATION_STRATEGY.md) | Evidence-bounded publication channels and canonical announcement copy. | +| [docs/PUBLICATION_LOG.md](docs/PUBLICATION_LOG.md) | Public GitHub calls, blocked external-channel attempts, and readiness boundary. | ## Verifying this repository diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md new file mode 100644 index 0000000..2fab721 --- /dev/null +++ b/docs/PUBLICATION_LOG.md @@ -0,0 +1,47 @@ +# Publication log + +This log records publication and outreach evidence. It does not upgrade a +readiness gate. Last verified: 2026-08-12. + +## GitHub public calls + +| Surface | Public artifact | Status | +|---|---|---| +| Draft implementation and review surface | [PR #8](https://github.com/thomaswillner/llm-errata/pull/8) | Open draft, targeting `agent/prod-readiness`. | +| Independent review request | [Issue #4](https://github.com/thomaswillner/llm-errata/issues/4) and its [exact review target](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269378077) | Open call for conformance, novelty, security, and distributed-systems review. | +| Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | +| Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | + +The published review target is source commit +[`35052868cde8b6f16fc6c20b4136ae0277f9a00e`](https://github.com/thomaswillner/llm-errata/commit/35052868cde8b6f16fc6c20b4136ae0277f9a00e) +with canonical Phase 2 surface digest +`a3133f759620ed942d6cd519d303c0be1554f0193b89e0a141c410aaf4c047b9`. +GitHub Actions completed successfully for [Python +3.11](https://github.com/thomaswillner/llm-errata/actions/runs/31616307869/job/94179861514) +and [Python +3.13](https://github.com/thomaswillner/llm-errata/actions/runs/31616307869/job/94179861561). + +## External channels + +| Channel | Current record | Evidence boundary | +|---|---|---| +| Hacker News | Attempted; authentication-blocked. No verified authenticated editor was available. | No public submission URL recorded. | +| LinkedIn | Attempted; authentication-blocked. Live signed-out state observed. | No public post URL recorded. | +| Reddit | Attempted; authentication-blocked. No verified authenticated editor was available. | No public submission URL recorded. | +| DEV Community | Attempted; authentication-blocked. No verified authenticated editor was available. | No public post URL recorded. | +| Medium | Attempted; authentication-blocked. No verified authenticated editor was available. | No public post URL recorded. | +| X | Attempted; authentication-blocked. Live signed-out state observed. | No public post URL recorded. | + +Zenodo and OSF remain deferred until a stable reviewed release. arXiv and +standards engagement remain deferred pending a paper-quality manuscript and +interoperability evidence. + +## Readiness boundary + +GitHub calls, invitations, CI results, and internal reviews are not independent +external evidence. Phase 2 remains incomplete: the explicit `errata quarantine` +CLI and vectors for key rotation, concurrent events, invalid targets, and +confidentiality remain absent; receipt state-root binding vectors exist but +receipt-binding coverage is partial. G2 remains `BLOCKED` pending internal Phase +2 completion and a dated independent external review of the complete +conformance surface. Repository verdict remains `NOT_PROD_READY`. From f91cb5bd4d5ac64ccef3f03102500c9aeca7776a Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:20:46 +0200 Subject: [PATCH 20/88] docs: record targeted review invitation --- docs/PUBLICATION_LOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 2fab721..278c61b 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -9,17 +9,20 @@ readiness gate. Last verified: 2026-08-12. |---|---|---| | Draft implementation and review surface | [PR #8](https://github.com/thomaswillner/llm-errata/pull/8) | Open draft, targeting `agent/prod-readiness`. | | Independent review request | [Issue #4](https://github.com/thomaswillner/llm-errata/issues/4) and its [exact review target](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269378077) | Open call for conformance, novelty, security, and distributed-systems review. | +| Targeted independent-review recruitment | [Inspeximus maintainer invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269444341) | Invitation to challenge the stated collision boundary; it is not external evidence. | | Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | -The published review target is source commit +The conformance review target remains source commit [`35052868cde8b6f16fc6c20b4136ae0277f9a00e`](https://github.com/thomaswillner/llm-errata/commit/35052868cde8b6f16fc6c20b4136ae0277f9a00e) with canonical Phase 2 surface digest `a3133f759620ed942d6cd519d303c0be1554f0193b89e0a141c410aaf4c047b9`. -GitHub Actions completed successfully for [Python -3.11](https://github.com/thomaswillner/llm-errata/actions/runs/31616307869/job/94179861514) +PR #8 currently heads at publication-log commit +[`cbcfd2a0d77f1060651bac781599edc469a42c26`](https://github.com/thomaswillner/llm-errata/commit/cbcfd2a0d77f1060651bac781599edc469a42c26). +GitHub Actions for that current PR head completed successfully for [Python +3.11](https://github.com/thomaswillner/llm-errata/actions/runs/31616716434/job/94181230842) and [Python -3.13](https://github.com/thomaswillner/llm-errata/actions/runs/31616307869/job/94179861561). +3.13](https://github.com/thomaswillner/llm-errata/actions/runs/31616716434/job/94181230861). ## External channels From a87b0c690a4b10b5daed02d66ae90396129a4218 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:27:56 +0200 Subject: [PATCH 21/88] docs: record targeted validation invitations --- docs/PUBLICATION_LOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 278c61b..dd1f63b 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -10,9 +10,14 @@ readiness gate. Last verified: 2026-08-12. | Draft implementation and review surface | [PR #8](https://github.com/thomaswillner/llm-errata/pull/8) | Open draft, targeting `agent/prod-readiness`. | | Independent review request | [Issue #4](https://github.com/thomaswillner/llm-errata/issues/4) and its [exact review target](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269378077) | Open call for conformance, novelty, security, and distributed-systems review. | | Targeted independent-review recruitment | [Inspeximus maintainer invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269444341) | Invitation to challenge the stated collision boundary; it is not external evidence. | +| Targeted adapter recruitment | [Remnic invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269523953) | Invitation to become a future independently authored adapter after Phase 2 completion; it is not technical evidence. | +| Targeted validator or system recruitment | [ai-memory-mcp invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269524142) | Invitation to choose one separate role: independently produced validator or independently operated system; it is not technical evidence. | +| Targeted standards/collision review | [Portable Agent Memory invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269524323) | Invitation to test the conformance/standards collision boundary; it is not review evidence. | | Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | +No targeted invitation has an acceptance or reply recorded. + The conformance review target remains source commit [`35052868cde8b6f16fc6c20b4136ae0277f9a00e`](https://github.com/thomaswillner/llm-errata/commit/35052868cde8b6f16fc6c20b4136ae0277f9a00e) with canonical Phase 2 surface digest From a00192b0c4da0ac1f1c274006b87ee9b780c9873 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:29:18 +0200 Subject: [PATCH 22/88] docs: record system nomination invitations --- docs/PUBLICATION_LOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index dd1f63b..1911717 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -13,6 +13,8 @@ readiness gate. Last verified: 2026-08-12. | Targeted adapter recruitment | [Remnic invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269523953) | Invitation to become a future independently authored adapter after Phase 2 completion; it is not technical evidence. | | Targeted validator or system recruitment | [ai-memory-mcp invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269524142) | Invitation to choose one separate role: independently produced validator or independently operated system; it is not technical evidence. | | Targeted standards/collision review | [Portable Agent Memory invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269524323) | Invitation to test the conformance/standards collision boundary; it is not review evidence. | +| Targeted operated-system nomination | [Mem0 invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990) | Invitation to nominate a future independently operated system; it is non-evidence and separate from validator authorship. | +| Targeted operated-system nomination | [Cognee invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156) | Invitation to nominate a future independently operated system; it is non-evidence and separate from validator authorship. | | Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | From ad46ef371aab16e5775e16fccc4695b994eb3c22 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:33:29 +0200 Subject: [PATCH 23/88] docs: record direct outreach status --- docs/PUBLICATION_LOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 1911717..38d6552 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -15,6 +15,7 @@ readiness gate. Last verified: 2026-08-12. | Targeted standards/collision review | [Portable Agent Memory invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269524323) | Invitation to test the conformance/standards collision boundary; it is not review evidence. | | Targeted operated-system nomination | [Mem0 invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990) | Invitation to nominate a future independently operated system; it is non-evidence and separate from validator authorship. | | Targeted operated-system nomination | [Cognee invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156) | Invitation to nominate a future independently operated system; it is non-evidence and separate from validator authorship. | +| Separate CODEOWNER volunteer recruitment | [PR #3 reviewer call](https://github.com/thomaswillner/llm-errata/pull/3#issuecomment-5269583214) | Invitation only; no volunteer or reviewer is recorded, no access was granted, and this call made no CODEOWNERS or protection change. It is not external evidence. | | Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | @@ -31,6 +32,17 @@ GitHub Actions for that current PR head completed successfully for [Python and [Python 3.13](https://github.com/thomaswillner/llm-errata/actions/runs/31616716434/job/94181230861). +## Direct email outreach + +| Recipient | Subject | Status | +|---|---|---| +| `founders@mem0.ai` | `Nomination request: Mem0 for an independent AI-memory interoperability experiment` | Sent 2026-08-12; no reply or acceptance recorded. Linked [Mem0 GitHub invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990) remains nomination-only. | +| `info@topoteretes.com` | `Nomination request: Cognee for an independent AI-memory interoperability experiment` | Sent 2026-08-12; no reply or acceptance recorded. Linked [Cognee GitHub invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156) remains nomination-only. | + +These messages are outreach, not external evidence. They do not authorize a +synthetic experiment, account access, data transfer, cost, integration, or a +role that overlaps validator authorship. + ## External channels | Channel | Current record | Evidence boundary | From 460883e093b52c99cec697443399da3e03ea2b40 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:39:39 +0200 Subject: [PATCH 24/88] docs: record discussion and security outreach --- docs/PUBLICATION_LOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 38d6552..96caa86 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -18,6 +18,7 @@ readiness gate. Last verified: 2026-08-12. | Separate CODEOWNER volunteer recruitment | [PR #3 reviewer call](https://github.com/thomaswillner/llm-errata/pull/3#issuecomment-5269583214) | Invitation only; no volunteer or reviewer is recorded, no access was granted, and this call made no CODEOWNERS or protection change. It is not external evidence. | | Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | +| GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | No targeted invitation has an acceptance or reply recorded. @@ -38,10 +39,11 @@ and [Python |---|---|---| | `founders@mem0.ai` | `Nomination request: Mem0 for an independent AI-memory interoperability experiment` | Sent 2026-08-12; no reply or acceptance recorded. Linked [Mem0 GitHub invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990) remains nomination-only. | | `info@topoteretes.com` | `Nomination request: Cognee for an independent AI-memory interoperability experiment` | Sent 2026-08-12; no reply or acceptance recorded. Linked [Cognee GitHub invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156) remains nomination-only. | +| `linzh@memtensor.cn` | `Independent security review invitation: post-export repair of derived AI memory` | Sent 2026-08-12 to the corresponding author contact for cited [arXiv:2604.16548](https://arxiv.org/html/2604.16548v1); preliminary security critique requested, with no reply or acceptance recorded. A formal qualifying review must wait for the final Phase 2 surface. | These messages are outreach, not external evidence. They do not authorize a synthetic experiment, account access, data transfer, cost, integration, or a -role that overlaps validator authorship. +role that overlaps validator authorship. They also do not satisfy G2. ## External channels From d467239787f74b7aa3a3f178b76037ec20b9394b Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:47:53 +0200 Subject: [PATCH 25/88] docs: bind publication log to current PR head --- docs/PUBLICATION_LOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 96caa86..73df311 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -27,11 +27,11 @@ The conformance review target remains source commit with canonical Phase 2 surface digest `a3133f759620ed942d6cd519d303c0be1554f0193b89e0a141c410aaf4c047b9`. PR #8 currently heads at publication-log commit -[`cbcfd2a0d77f1060651bac781599edc469a42c26`](https://github.com/thomaswillner/llm-errata/commit/cbcfd2a0d77f1060651bac781599edc469a42c26). +[`460883e093b52c99cec697443399da3e03ea2b40`](https://github.com/thomaswillner/llm-errata/commit/460883e093b52c99cec697443399da3e03ea2b40). GitHub Actions for that current PR head completed successfully for [Python -3.11](https://github.com/thomaswillner/llm-errata/actions/runs/31616716434/job/94181230842) +3.11](https://github.com/thomaswillner/llm-errata/actions/runs/31619022097/job/94188870893) and [Python -3.13](https://github.com/thomaswillner/llm-errata/actions/runs/31616716434/job/94181230861). +3.13](https://github.com/thomaswillner/llm-errata/actions/runs/31619022097/job/94188870819). ## Direct email outreach From bc25c35dcc8a7985a4f26876b510ba4ac147de7d Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:48:24 +0200 Subject: [PATCH 26/88] docs: make publication checkpoint non-self-referential --- docs/PUBLICATION_LOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 73df311..8b5bf4d 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -26,12 +26,15 @@ The conformance review target remains source commit [`35052868cde8b6f16fc6c20b4136ae0277f9a00e`](https://github.com/thomaswillner/llm-errata/commit/35052868cde8b6f16fc6c20b4136ae0277f9a00e) with canonical Phase 2 surface digest `a3133f759620ed942d6cd519d303c0be1554f0193b89e0a141c410aaf4c047b9`. -PR #8 currently heads at publication-log commit +The implementation and outreach surface at commit [`460883e093b52c99cec697443399da3e03ea2b40`](https://github.com/thomaswillner/llm-errata/commit/460883e093b52c99cec697443399da3e03ea2b40). -GitHub Actions for that current PR head completed successfully for [Python +was the fixed head exercised by GitHub Actions successfully on [Python 3.11](https://github.com/thomaswillner/llm-errata/actions/runs/31619022097/job/94188870893) and [Python 3.13](https://github.com/thomaswillner/llm-errata/actions/runs/31619022097/job/94188870819). +The live [PR #8](https://github.com/thomaswillner/llm-errata/pull/8) identifies +later publication-log-only commits and their own CI runs without requiring a +self-referential commit identifier in this file. ## Direct email outreach From 27da4d09c79f03501b1302b9931be3561ff05e75 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:56:23 +0200 Subject: [PATCH 27/88] docs: qualify production cryptography candidates --- CHANGELOG.md | 5 ++ PRODUCTION_READINESS.md | 2 +- SOURCES.md | 19 ++++ THREAT_MODEL.md | 7 +- docs/CRYPTOGRAPHY_QUALIFICATION.md | 130 ++++++++++++++++++++++++++++ readiness/production-readiness.json | 5 +- 6 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 docs/CRYPTOGRAPHY_QUALIFICATION.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c91c0..a6137f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Added a fail-closed production-cryptography qualification record. PyCA + reproduced the repository's RFC 8032 vectors but documents no external + project audit; libsodium has a published assessment for older releases, not + the current 1.0.22 build. G3 remains `BLOCKED` pending exact-build + qualification, lifecycle implementation, and independent security review. - Added provider-neutral semantic probes, deterministic recorded fixtures, and `errata semantic-test` documentation as Phase 2 item 6 internal implementation. Opened independent review, independent implementation, and Phase 3 system-nomination programs. G2 remains `BLOCKED`: repository artifacts and local tests do not substitute for dated independent external review. - Added a fail-closed production-readiness ledger and human verification matrix. Local green tests cannot produce `PROD_READY`; every required gate must pass, and external gates require dated evidence naming an independent producer. - Added drift checks binding README maturity and SECURITY support policy to `VERSION`. The current verdict remains `NOT_PROD_READY`: only G1, document consistency, is complete. diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 5d2b181..472938e 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -16,7 +16,7 @@ Build health means the repository's deterministic structure/metadata checks, bou |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | | G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded. | Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface. | -| G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` records current security scope; no qualifying production cryptography evidence is recorded. | Dated independent security review and evidence of audited production signer, rotation, recovery, and delegation. | +| G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | | G6 | Security, observability, recovery, compatibility, performance, deployment, and rollback gates pass with current operational evidence. | `BLOCKED` | `SECURITY.md` records reporting scope; no qualifying production operations evidence is recorded. | Current operational evidence covering deployment, rollback, recovery, observability, privacy, compatibility, load, denial-of-service, and incident response. | diff --git a/SOURCES.md b/SOURCES.md index e9b5edc..0f7967f 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -99,3 +99,22 @@ An archived copy of the Glean report exists at - When a source disappears, cite the archive snapshot and say so. If no snapshot exists, the correct action is to narrow the affected row, not to keep the citation. + +## Production-cryptography candidate sources + +These sources were read on 2026-08-12 for the internal candidate assessment in +[`docs/CRYPTOGRAPHY_QUALIFICATION.md`](docs/CRYPTOGRAPHY_QUALIFICATION.md). +They are security-maintenance and compatibility evidence, not independent +review of LLM Errata and not evidence that G3 passes. + +| Source | Pin or version | Verified 2026-08-12 | +|---|---|---| +| [PyCA Ed25519 API](https://cryptography.io/en/latest/hazmat/primitives/asymmetric/ed25519/) | live documentation; repository `main` observed at `95c04524ef533d6173e4c61d5781ea34bff0e768` | Raw 32-byte private/public key loading, 64-byte signatures, and one-shot verification are supported | +| [PyCA security policy](https://cryptography.io/en/latest/security/) | live documentation | Most recent release and `main` receive security support; binary releases are refreshed for OpenSSL security updates | +| [PyCA project documentation](https://cryptography.io/en/latest/) | live documentation | Explicitly says project code and documentation have not undergone an external audit | +| [PyCA on PyPI](https://pypi.org/project/cryptography/50.0.0/) | 50.0.0 | Python requirement and Apache-2.0 OR BSD-3-Clause licence expression confirmed | +| [OpenSSL Ed25519](https://docs.openssl.org/3.5/man7/EVP_SIGNATURE-ED25519/) | OpenSSL 3.5 documentation | One-shot RFC 8032 Ed25519 signing and verification plus raw key loading confirmed | +| [libsodium signatures](https://doc.libsodium.org/public-key_cryptography/public-key_signatures) | live documentation; stable branch observed at `701aa826b97dc84a353d70a551d49dc26da539c5` | Seed keypair, detached signature, detached verification, exact single-part Ed25519 algorithm, and key sizes confirmed | +| [libsodium 1.0.22](https://github.com/jedisct1/libsodium/releases/tag/1.0.22-RELEASE) | 1.0.22, 2026-04-09 | Current public point release at assessment time | +| [Libsodium 1.0.12 and 1.0.13 Security Assessment](https://www.privateinternetaccess.com/blog/wp-content/uploads/2017/08/libsodium.pdf) | assessment of 1.0.12 and 1.0.13 | Third-party review included Ed25519 signatures and reported no major vulnerabilities in reviewed versions; it is not a current 1.0.22 audit | +| [CVE-2025-69277](https://nvd.nist.gov/vuln/detail/CVE-2025-69277) | CVE-2025-69277 | Older libsodium revisions had an Ed25519 point-validation flaw; 1.0.22 post-dates the cited fix, without making vulnerability review unnecessary | diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 21d5cf3..9c05dc7 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -84,8 +84,11 @@ repair. Nothing rate-limits or prioritises. The bundled Ed25519 is a reference implementation: correct against the RFC 8032 vectors, and **not constant-time**. Python integer arithmetic cannot be. It is -appropriate where signing keys are not attacker-facing. A deployment should -link libsodium and substitute a `Signer`; no caller changes. +appropriate where signing keys are not attacker-facing. A deployment must +substitute a qualified production `Signer`; no caller changes. The current +candidate assessment in +[`docs/CRYPTOGRAPHY_QUALIFICATION.md`](docs/CRYPTOGRAPHY_QUALIFICATION.md) +does not yet qualify either reviewed backend or pass readiness gate G3. ## Reporting diff --git a/docs/CRYPTOGRAPHY_QUALIFICATION.md b/docs/CRYPTOGRAPHY_QUALIFICATION.md new file mode 100644 index 0000000..2856760 --- /dev/null +++ b/docs/CRYPTOGRAPHY_QUALIFICATION.md @@ -0,0 +1,130 @@ +# Production cryptography qualification + +**Assessment date:** 2026-08-12 + +**Repository verdict:** `NOT_PROD_READY` + +**Readiness gate:** G3 remains `BLOCKED` + +## Decision + +Do not replace the reference signer yet. + +PyCA `cryptography` and libsodium are both technically credible Ed25519 +backends, but the evidence reviewed here does not prove the complete G3 +criterion: an audited, side-channel-resistant production signer plus an +independent review of key generation, custody, rotation, delegation, recovery, +revocation, compromise handling, and deployment. + +The leading path is to qualify libsodium 1.0.22 through a small Python binding +or an existing maintained binding, subject to current-version delta review and +the lifecycle design below. PyCA remains a useful interoperability oracle, not +the selected G3 implementation, because its current documentation explicitly +states that the project has not undergone an external audit. + +## Evidence collected + +### Reference baseline + +The repository's `prototype/ed25519.py` is pure Python. It reproduces the +checked RFC 8032 vectors, but Python big-integer operations are not a suitable +side-channel boundary for production signing. The `Signer` protocol in +`prototype/signing.py` isolates replacement from controller and receipt logic. + +### PyCA `cryptography` + +| Item | Observed evidence | +|---|---| +| Installed test version | 49.0.0 | +| Current PyPI release | 50.0.0 | +| Python constraint | `!=3.9.0,!=3.9.1,>=3.9` | +| Licence | Apache-2.0 OR BSD-3-Clause | +| Local backend | OpenSSL 4.0.1; FIPS mode disabled | +| Compatibility | All five checked RFC 8032 vectors produced identical raw public keys and signatures; verification passed | +| Security boundary | PyCA delegates cryptographic operations to OpenSSL | +| Audit status | PyCA documentation says its code and documentation have not undergone an external audit | + +Compatibility proves that existing 32-byte seeds, 32-byte public keys, and +64-byte signatures can retain their wire representation. It does not prove +constant-time behavior, safe key custody, a supported deployment build, or G3. + +### libsodium + +| Item | Observed evidence | +|---|---| +| Local library | 1.0.22 | +| Current release | 1.0.22, published 2026-04-09 | +| Candidate API | `crypto_sign_seed_keypair`, `crypto_sign_detached`, and `crypto_sign_verify_detached` | +| Wire model | Single-part API is Ed25519; seed, public key, and detached signature sizes match the existing profile | +| Published assessment | Independent assessment covered 1.0.12 and 1.0.13, including Ed25519 signatures, and reported no major vulnerabilities in reviewed versions | +| Current-version gap | The 2017 assessment is not an audit of 1.0.22; a documented delta and build-specific review are still required | +| Security maintenance | 1.0.22 post-dates the fix for CVE-2025-69277; this fact does not replace vulnerability scanning or review | + +The assessment makes libsodium the stronger audited-lineage candidate. It does +not justify describing 1.0.22 or an application binding as independently +audited without reviewing changes since 1.0.13 and the exact production build. + +## Required production design + +A qualifying implementation must preserve these contracts: + +1. Use raw Ed25519 seeds and public keys without changing canonical payload + bytes, signatures, receipt schema, or key identifiers. +2. Generate production seeds from an operating-system CSPRNG or approved key + service. Test fixture labels may remain deterministic; production labels + must never be converted into keys. +3. Keep private keys outside receipts, logs, command output, test artifacts, + and repository files. Define process-memory exposure and zeroization limits. +4. Bind every erratum and receipt to an active key identifier and declared key + validity interval. +5. Represent rotation, delegation, revocation, and recovery as authenticated, + sequenced lifecycle events. An importer must fail closed on unknown, + expired, revoked, conflicting, or rollback key state. +6. Preserve the old verification key for historical receipts while preventing + it from authorizing new events after its terminal sequence. +7. Define compromise recovery independently from ordinary rotation. Recovery + must identify its trust anchor and cannot be authorized only by the + compromised key. +8. Pin the library and binding version, record linked library identity, scan + known vulnerabilities, and reproduce RFC plus repository receipt vectors in + every supported build. + +## Independent review request + +A qualifying reviewer should answer, against an exact commit and build: + +- Does the chosen Ed25519 path avoid secret-dependent control flow and memory + access for the supported targets, or clearly identify any residual channel? +- Does the binding preserve raw seed/public/signature compatibility and reject + malformed or non-canonical signatures consistently? +- Can any rotation, delegation, recovery, replay, sequence-gap, or key-ID + ambiguity authorize an attacker or strand legitimate historical evidence? +- Are private-key generation, storage, process exposure, backup, recovery, + destruction, logging, and incident procedures explicit and testable? +- Are library, binding, compiler, platform, and vulnerability evidence pinned + closely enough to support the production claim? + +The report must disclose reviewer identity, relationship and conflicts, date, +reviewed commit, build inputs, scope, tests, findings, residual risks, and +verdict. Invitation, compatibility testing, CI, or this internal assessment is +not independent security evidence. + +## Reproduction record + +The compatibility probe loaded the five RFC 8032 vectors already published in +`tests/test_ed25519.py`, constructed PyCA private keys from each 32-byte seed, +and compared raw public keys and signatures with literal expected values. It +then verified every signature with the corresponding raw public key. + +Observed result: + +```text +pyca_rfc8032_compatibility=PASS vectors=5 +installed_cryptography=49.0.0 +openssl_version=OpenSSL 4.0.1 9 Jun 2026 +fips_enabled=False +``` + +This record is local compatibility evidence only. A future implementation must +turn the probe into checked-in tests and run it across every declared platform +before the backend changes. diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index 261205b..5ec01c5 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -48,7 +48,10 @@ "class": "external", "status": "BLOCKED", "criterion": "Production signing uses an audited constant-time library through the Signer seam and independent security review covers key lifecycle.", - "evidence": [{"kind": "repository", "ref": "THREAT_MODEL.md"}] + "evidence": [ + {"kind": "repository", "ref": "THREAT_MODEL.md"}, + {"kind": "repository", "ref": "docs/CRYPTOGRAPHY_QUALIFICATION.md"} + ] }, { "id": "G4", From 01918c5c891f93304bb777ff3a63327d44e094e9 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 18:58:17 +0200 Subject: [PATCH 28/88] docs: record cryptography review request --- docs/PUBLICATION_LOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 8b5bf4d..d753634 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -13,6 +13,7 @@ readiness gate. Last verified: 2026-08-12. | Targeted adapter recruitment | [Remnic invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269523953) | Invitation to become a future independently authored adapter after Phase 2 completion; it is not technical evidence. | | Targeted validator or system recruitment | [ai-memory-mcp invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269524142) | Invitation to choose one separate role: independently produced validator or independently operated system; it is not technical evidence. | | Targeted standards/collision review | [Portable Agent Memory invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269524323) | Invitation to test the conformance/standards collision boundary; it is not review evidence. | +| Targeted production-cryptography review | [Exact-build security-review request](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269850937) | Published 2026-08-12 against qualification commit `27da4d09c79f03501b1302b9931be3561ff05e75`; requests independent side-channel, build, binding, and key-lifecycle review. Invitation only: G3 remains `BLOCKED`. | | Targeted operated-system nomination | [Mem0 invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990) | Invitation to nominate a future independently operated system; it is non-evidence and separate from validator authorship. | | Targeted operated-system nomination | [Cognee invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156) | Invitation to nominate a future independently operated system; it is non-evidence and separate from validator authorship. | | Separate CODEOWNER volunteer recruitment | [PR #3 reviewer call](https://github.com/thomaswillner/llm-errata/pull/3#issuecomment-5269583214) | Invitation only; no volunteer or reviewer is recorded, no access was granted, and this call made no CODEOWNERS or protection change. It is not external evidence. | From c58cc2e499d40bf00598aec3f04e712ae764b975 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 19:05:46 +0200 Subject: [PATCH 29/88] docs: record cryptography reviewer outreach --- docs/PUBLICATION_LOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index d753634..5c52da2 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -44,6 +44,8 @@ self-referential commit identifier in this file. | `founders@mem0.ai` | `Nomination request: Mem0 for an independent AI-memory interoperability experiment` | Sent 2026-08-12; no reply or acceptance recorded. Linked [Mem0 GitHub invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990) remains nomination-only. | | `info@topoteretes.com` | `Nomination request: Cognee for an independent AI-memory interoperability experiment` | Sent 2026-08-12; no reply or acceptance recorded. Linked [Cognee GitHub invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156) remains nomination-only. | | `linzh@memtensor.cn` | `Independent security review invitation: post-export repair of derived AI memory` | Sent 2026-08-12 to the corresponding author contact for cited [arXiv:2604.16548](https://arxiv.org/html/2604.16548v1); preliminary security critique requested, with no reply or acceptance recorded. A formal qualifying review must wait for the final Phase 2 surface. | +| `mgreen@cs.jhu.edu` | `Independent review inquiry: libsodium audit lineage for LLM Errata` | Sent 2026-08-12 to Matthew Green using the professional address published in his Johns Hopkins CV. Requested availability or referral for a scoped review of libsodium 1.0.22 audit delta, Ed25519 binding/build, side channels, and key lifecycle against exact qualification commit `27da4d09c79f03501b1302b9931be3561ff05e75`. Availability/scope inquiry only; no engagement, cost, access, or evidence established. | +| `contact@edgesecurity.com` | `Scope inquiry: independent libsodium integration and key-lifecycle review` | Sent 2026-08-12 using the address on Edge Security's official site after official libsodium documentation identified the company for cryptographic and implementation auditing of libsodium usage. Requested availability and scope/cost discussion against exact qualification commit `27da4d09c79f03501b1302b9931be3561ff05e75`. No engagement, cost, access, or evidence established. | These messages are outreach, not external evidence. They do not authorize a synthetic experiment, account access, data transfer, cost, integration, or a From 9d2b8cadc56835873a5a1eca2bcf4f8e5673631e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 19:10:13 +0200 Subject: [PATCH 30/88] docs: publish CODEOWNER reviewer intake --- docs/PUBLICATION_LOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 5c52da2..44bcdd7 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -17,6 +17,7 @@ readiness gate. Last verified: 2026-08-12. | Targeted operated-system nomination | [Mem0 invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990) | Invitation to nominate a future independently operated system; it is non-evidence and separate from validator authorship. | | Targeted operated-system nomination | [Cognee invitation](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156) | Invitation to nominate a future independently operated system; it is non-evidence and separate from validator authorship. | | Separate CODEOWNER volunteer recruitment | [PR #3 reviewer call](https://github.com/thomaswillner/llm-errata/pull/3#issuecomment-5269583214) | Invitation only; no volunteer or reviewer is recorded, no access was granted, and this call made no CODEOWNERS or protection change. It is not external evidence. | +| Dedicated CODEOWNER reviewer intake | [Issue #10](https://github.com/thomaswillner/llm-errata/issues/10) and [PR #3 linkage](https://github.com/thomaswillner/llm-errata/pull/3#issuecomment-5269972483) | Public volunteer intake for exact readiness-foundation commit `2d086611fed84fc542fda9d2ccc541530c0731f3`. Requires identity, background, relationship/conflict disclosure, and availability before the owner separately considers limited access or a reviewed CODEOWNERS amendment. Created 2026-08-12; grants no access and is not approval or external evidence. | | Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | | GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | From 5e42215188ab766151312e1a6bc9aeb30a20a487 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 21:47:30 +0200 Subject: [PATCH 31/88] docs: specify quarantine and operational evidence gates --- ...26-08-12-g6-operational-evidence-design.md | 178 ++++++++++++++++++ ...2026-08-12-quarantine-checkpoint-design.md | 160 ++++++++++++++++ 2 files changed, 338 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-12-g6-operational-evidence-design.md create mode 100644 docs/superpowers/specs/2026-08-12-quarantine-checkpoint-design.md diff --git a/docs/superpowers/specs/2026-08-12-g6-operational-evidence-design.md b/docs/superpowers/specs/2026-08-12-g6-operational-evidence-design.md new file mode 100644 index 0000000..0b88227 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-g6-operational-evidence-design.md @@ -0,0 +1,178 @@ +# G6 Operational Evidence Design + +**Date:** 2026-08-12 + +**Status:** Approved for readiness-checker implementation + +## Purpose + +Turn G6 from a broad prose aspiration into a fail-closed operational evidence +contract. G6 remains one external gate. It passes only when one independently +produced structured report binds an exact repository commit and deployment to +measured evidence for every mandatory operational scope. + +Internal tests validate the contract; they do not satisfy it. Missing operator +thresholds, synthetic example reports, repository CI, invitations, and author- +or agent-produced assessments keep G6 `BLOCKED`. + +## Mandatory scopes + +One report contains exactly one result for each unique scope token: + +1. `deployment-integrity-provenance` — deployed artifact identity, build and + dependency provenance, signature or digest verification, and environment + binding; +2. `rollback-exercise` — measured rollback from the tested deployment and + verification of restored service and state; +3. `backup-recovery-rto-rpo` — recovery exercise with declared numeric RTO and + RPO thresholds and measured recovery time and recovery-point loss; +4. `lifecycle-observability-alerting` — observable quarantine, rebuild, test, + attest, refusal, and incomplete-coverage lifecycle with alert delivery; +5. `telemetry-privacy-redaction` — verification that logs, traces, metrics, and + retained evidence exclude prohibited memory content, secrets, and erased + values; +6. `supported-version-compatibility` — declared supported runtime, schema, + adapter, store, and platform versions exercised against a compatibility + matrix; +7. `representative-latency-throughput` — declared workload with measured + latency and throughput against numeric thresholds; +8. `overload-rate-limit-dos` — overload, rate limiting, resource exhaustion, + and denial-of-service behavior measured against declared safety thresholds; +9. `incident-response-exercise` — dated detection, containment, recovery, + communication, and learning exercise with measured response thresholds; and +10. `operational-access-secrets-dependencies-vulnerability-management` — + least-privilege access, secret lifecycle, dependency inventory, supported + patch policy, and vulnerability response measured against declared gates. + +Scope tokens are normative and unique. Aliases, duplicate tokens, missing +tokens, or extra tokens fail qualification rather than being guessed into a +mapping. + +## Report envelope + +A qualifying evidence entry is `kind: external` and includes: + +- `ref`: non-generic HTTPS report URL with a path, or a scoped URN; +- `producer`, `producer_identity`, `relationship`, `conflicts`, and the exact + attestation `llm-errata-independent-operational-review-v1`; +- `observed` ISO date and `result` equal to `pass`, `pass-with-findings`, or + `fail`; +- `reviewed_commit`: full 40-character lowercase commit available in repository + history; +- `surface_digest`: canonical digest of G6 readiness and operational contract + files at that commit and in current checkout; +- `deployment`: stable deployment ID, platform, environment, artifact digest, + build provenance reference, and deployment timestamp; +- `workload`: name, dataset class, synthetic-data declaration, request or event + volume, concurrency, duration, and failure domain; +- `observation_window`: ISO start and end timestamps with end after start; and +- `scopes`: the ten structured results. + +The producer must be independent of author, owner, maintainer, reference +implementer, deployment operator being assessed, and project-controlled agents. +Declared conflicts are retained for review; an empty list is permitted, but a +missing conflicts field is not. + +## Per-scope measurement contract + +Every scope result contains: + +- its unique `scope` token; +- `status`: `pass` or `fail`; +- at least one raw `artifact` reference; +- non-empty `measurements`; and +- optional findings that cannot override a failed comparator. + +Every measurement declares: + +- `metric`: stable non-empty name; +- `value`: finite JSON number; +- `unit`: non-empty explicit unit; +- `comparator`: one of `<=`, `<`, `>=`, `>`, `==`, or `!=`; +- `threshold`: finite JSON number in the same unit; and +- `evidence_ref`: raw evidence URL or scoped URN. + +The checker evaluates every comparator. A scope passes only when its declared +status is `pass`, every measurement comparator passes, and required identity, +artifact, workload, platform, failure-domain, and observation-window fields are +present. Text such as “acceptable,” absent numeric thresholds, mixed units, +NaN, infinity, or a generic evidence URL cannot pass. + +Operator-specific policy decides threshold values; the repository does not +invent universal latency, RTO, RPO, volume, or incident thresholds. A report +without them remains structurally valid evidence of work but cannot qualify G6 +for `PASS`. + +## Gate qualification + +`valid_g6_operational_evidence()` validates complete report structure and exact +commit/deployment binding. `qualifying_g6_operational_evidence()` additionally +requires: + +- report result `pass` or `pass-with-findings`; +- all ten scopes present exactly once; +- every scope status `pass`; +- every comparator true; and +- current checkout G6 surface digest equal to both reported digest and the + digest reconstructed from `reviewed_commit`. + +Generic external evidence remains insufficient. G6 changes to `PASS` only when +at least one evidence entry qualifies. A structurally complete `fail` report is +valuable evidence but keeps G6 from passing. + +## Canonical G6 surface + +The digest covers: + +- `docs/OPERATIONAL_READINESS.md`; +- `PRODUCTION_READINESS.md`; +- `SECURITY.md`; +- `ROADMAP.md`; +- `readiness/production-readiness.json`; +- `scripts/check_readiness.py`; and +- `tests/test_readiness.py`. + +It uses the same ordered `relative path + NUL + raw bytes + NUL` SHA-256 framing +as G2. A missing file, unavailable commit, or byte drift blocks qualification. + +## Human documentation + +`docs/OPERATIONAL_READINESS.md` explains the ten scopes, report schema, +measurement rules, independence boundary, operator workflow, and example +non-qualifying skeleton. It contains no fabricated measurements or thresholds. + +The readiness ledger and human matrix cite the document and state exactly why +G6 is `BLOCKED`: no independent report binding an exact commit, deployment, and +passing measured comparator for all ten scopes exists. `CHANGELOG.md` records +the strengthened contract, and `SOURCES.md` pins the primary grounding sources: + +- NIST SP 800-61 Revision 3; +- NIST SP 800-218; +- SLSA version 1.2; +- OpenTelemetry signals and semantic conventions; and +- OWASP Logging Cheat Sheet. + +Sources support scope selection and terminology. They do not certify this +project or supply operator thresholds. + +## Test contract + +Strict TDD must prove rejection of: + +- generic URL-only evidence; +- internal or ambiguous producer; +- missing or invalid identity, conflicts, relationship, or attestation; +- nonexistent or non-current commit and surface digest drift; +- missing deployment, platform, artifact, provenance, workload, failure domain, + or observation window; +- missing, duplicate, unknown, or extra scope tokens; +- missing artifacts or measurements; +- missing, nonnumeric, nonfinite, unitless, or invalid thresholds and values; +- invalid comparator and failed comparator; +- a scope marked pass when one comparator fails; +- report result `fail`; and +- nine passing scopes presented as complete evidence. + +Positive tests use a complete synthetic report bound to a temporary committed +repository copy. Test reports do not enter the production-readiness ledger and +do not upgrade G6. diff --git a/docs/superpowers/specs/2026-08-12-quarantine-checkpoint-design.md b/docs/superpowers/specs/2026-08-12-quarantine-checkpoint-design.md new file mode 100644 index 0000000..640b593 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-quarantine-checkpoint-design.md @@ -0,0 +1,160 @@ +# Quarantine Checkpoint Design + +**Date:** 2026-08-12 + +**Status:** Approved for Phase 2 implementation + +## Purpose + +Make the CLI's quarantine-before-repair guarantee durable and independently +inspectable. `errata quarantine` authenticates exactly the next pending erratum, +gates the complete known descendant closure across every declared retrieval +adapter, and writes an atomic checkpoint. `errata repair` refuses to mutate +state without the matching unconsumed checkpoint. + +The existing `Importer.repair()` method remains the Phase 1 in-process atomic +composition. The explicit checkpoint is required at the Phase 2 CLI boundary; +this change does not weaken or silently replace the in-process API. + +## Checkpoint model + +`prototype/checkpoints.py` owns a strict immutable `QuarantineCheckpoint` model +and its persistence. A checkpoint binds: + +- schema version `1`; +- erratum ID and sequence; +- target root; +- inspectable pre-state root; +- every gated artifact grouped by adapter; +- every declared adapter, its required flag, and its coverage limitation; +- creation timestamp; +- consumed state and optional consumption timestamp; and +- a canonical SHA-256 checkpoint digest. + +The digest is computed over canonical JSON with sorted object keys, compact +separators, UTF-8 encoding, and the `checkpoint_digest` field omitted. Lists +retain semantic order. The consumed fields are excluded from the identity +digest so consumption does not change which quarantine operation the +checkpoint proves; atomic persistence still protects their current value. + +Checkpoint paths are deterministic: +`checkpoints/-.json`. Erratum IDs are already schema +constrained; path construction additionally rejects separators and traversal. + +## Quarantine command + +`errata quarantine` performs one bounded operation: + +1. load the signed feed and identify exactly the next sequence after the + workspace's last applied sequence; +2. authenticate that erratum with the configured owner verification key and + root registry; +3. reject a sequence gap, invalid target, missing lineage root, replay, or an + existing consumed checkpoint; +4. capture the inspectable pre-state root; +5. enumerate the target's known descendant closure through each adapter; +6. gate every enumerable descendant before reporting completion; +7. record required opaque or non-enumerable stores as `unknown`, with no gated + artifacts and an explicit limitation; +8. persist the checkpoint atomically using a same-directory temporary file, + file flush and `fsync`, `os.replace`, and directory `fsync`; and +9. print the checkpoint path and digest. + +If any enumerable retrieval adapter fails to gate its complete returned set, +the command fails closed and does not publish a successful checkpoint. State +already gated by an interrupted attempt remains gated; rerunning the command +re-authenticates the erratum and may replace only an identical unconsumed +checkpoint after the current state is reconciled against its bound pre-state. + +Opaque required stores do not make quarantine appear complete. They remain +`unknown` and flow into the eventual receipt limitations. + +## Repair command + +`errata repair` processes exactly the next pending erratum and requires the +deterministic checkpoint path. Before rebuild it: + +1. re-authenticates the signed erratum; +2. validates the checkpoint schema and recomputes its canonical digest; +3. verifies erratum ID, sequence, target root, pre-state root, adapter inventory, + gated artifact set, coverage limitations, and unconsumed state; +4. confirms every recorded enumerable artifact is still gated and rejects + workspace or target drift; and +5. starts rebuild only after all checks pass. + +Missing, malformed, replayed, consumed, wrong-target, wrong-sequence, +wrong-pre-state, digest-mismatched, adapter-drifted, or ungated checkpoints are +refused without rebuild, receipt, or applied-sequence advancement. + +Repair uses the checkpoint's already-gated artifact map and opaque-store +limitations. It does not run quarantine a second time. It runs rebuild, the +repair triad, signed attest, receipt writeback, and applied-state writeback in +that order. Only after both durable writebacks succeed is the checkpoint marked +consumed atomically. A failure before consumption retains the unconsumed +checkpoint and gated state for safe resume. + +Because receipt or applied-state writeback can fail between their two durable +operations, repair reconciliation must be idempotent: an existing byte-identical +receipt is accepted, and applied state may advance only to the same target and +sequence. Contradictory durable state is refused. + +## Workspace and controller boundaries + +`Workspace` owns checkpoint paths and atomic file operations. `Importer` gains +focused public operations for authenticated quarantine and repair from an +already validated checkpoint. Private adapter details remain inside the +controller; CLI code orchestrates persistence, not repair semantics. + +The checkpoint records adapter names, required flags, gated artifact IDs, and +limitations. An adapter added, removed, renamed, or changed from optional to +required between quarantine and repair is material drift and blocks repair. + +## CLI behavior + +- `errata quarantine`: exit `0` after a durable valid checkpoint; exit `1` for + authentication, sequencing, target, gating, checkpoint, or state failure. +- `errata repair`: exit `0` only for aggregate `verified`; exit `2` for an + honestly completed `partial` or `unknown` repair; exit `1` when checkpoint or + repair admission fails. +- `errata pull` continues to report unapplied feed entries. +- `make cli-demo` invokes `quarantine` before `repair`. + +The command never accepts a caller-supplied checkpoint path. Deterministic +workspace resolution prevents substituting evidence from another target. + +## Security and recovery properties + +- Signed input is authenticated both before gating and before rebuild. +- A checkpoint is target-, sequence-, state-, adapter-, and artifact-bound. +- Canonical digests detect content mutation, not filesystem metadata changes. +- Atomic replacement prevents a partially written checkpoint from authorizing + repair. +- Consumption is last, so interrupted rebuild or writeback remains resumable. +- Replay and cross-target substitution fail closed. +- Opaque coverage never becomes verified. +- Checkpoints contain artifact identifiers and commitments, not erased values. + +This remains a reference implementation. Production deployment still requires +audited constant-time cryptography, hardened filesystem ownership and locking, +operator authentication, and independent review. + +## Test contract + +Strict TDD covers: + +- exactly-next authenticated erratum selection; +- complete descendant gating before checkpoint persistence; +- deterministic canonical digest and atomic round trip; +- opaque required adapter recorded as `unknown`; +- missing, malformed, mutated, consumed, replayed, wrong-target, + wrong-sequence, pre-state-drifted, adapter-drifted, and ungated checkpoint + refusal; +- repair starting only from a matching checkpoint; +- checkpoint retained after interrupted rebuild or writeback; +- checkpoint consumed only after receipt and applied-state durability; +- correction, supersession, and erasure behavior; +- negative, positive, and preservation triad preservation; and +- CLI demo ordering. + +All fixtures remain synthetic and `make check` remains offline and +standard-library-only. From aff63fcbdb6d163896d42886841c48ba42295f09 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 21:49:33 +0200 Subject: [PATCH 32/88] docs: plan quarantine and G6 implementation --- .../2026-08-12-g6-operational-evidence.md | 146 +++++++++++++++ .../plans/2026-08-12-quarantine-checkpoint.md | 167 ++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-12-g6-operational-evidence.md create mode 100644 docs/superpowers/plans/2026-08-12-quarantine-checkpoint.md diff --git a/docs/superpowers/plans/2026-08-12-g6-operational-evidence.md b/docs/superpowers/plans/2026-08-12-g6-operational-evidence.md new file mode 100644 index 0000000..805477c --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-g6-operational-evidence.md @@ -0,0 +1,146 @@ +# G6 Operational Evidence Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make G6 pass only from one independent, commit- and deployment-bound report with passing measured evidence for all ten operational scopes. + +**Architecture:** `scripts/check_readiness.py` adds strict G6 report validators and a canonical G6 surface digest, then specializes external-pass qualification by gate. Human docs define operator-owned thresholds without fabricating values. Ledger stays `BLOCKED` until genuine evidence arrives. + +**Tech Stack:** Python 3.11+ standard library, JSON, SHA-256, Git object inspection, `unittest`, Markdown. + +## Global Constraints + +- Keep one gate ID: `G6`. +- Require exactly ten unique normative scope tokens. +- Every measurement has finite value and threshold, one unit, a supported comparator, and raw evidence reference. +- Missing thresholds remain `BLOCKED`. +- Internal producers, repository CI, invitations, and example reports never qualify. +- Exact current checkout and reviewed-commit surface digests must match. +- Current verdict remains `NOT_PROD_READY` without genuine external evidence. + +--- + +### Task 1: G6 evidence validator and digest + +**Files:** +- Modify: `scripts/check_readiness.py` +- Modify: `tests/test_readiness.py` + +**Interfaces:** +- Produces: `G6_SCOPE`, `G6_ATTESTATION`, `g6_surface_files`, `g6_surface_digest`, `g6_surface_digest_at_commit`, `valid_g6_operational_evidence`, and `qualifying_g6_operational_evidence`. +- Extends gate validation so G2 uses G2 qualification, G6 uses G6 qualification, and other external gates retain generic independent evidence behavior. + +- [ ] **Step 1: Write a complete synthetic report fixture helper** + + In `tests/test_readiness.py`, build ten literal scope results with hand-derived passing measurements. Commit a temporary repository copy before binding its full commit and surface digest. + +- [ ] **Step 2: Write failing positive and mutation tests** + + Prove a complete report validates and qualifies. Mutate one field per test to reject generic URL, producer identity, relationship, conflicts, attestation, commit, digest, deployment identity, platform, artifact digest, provenance, workload, volume, concurrency, duration, failure domain, observation window, missing/duplicate/extra scope, artifacts, measurements, value, unit, comparator, threshold, evidence ref, failed comparator, scope status, and report result. + +- [ ] **Step 3: Verify RED** + + Run `python3 -m unittest tests.test_readiness -v`. Expected: imports for G6 symbols fail. + +- [ ] **Step 4: Implement strict validation** + + Use exact-key validation for report subobjects where forward ambiguity would be unsafe. Reject `bool` as numeric, reject `math.isfinite(value) is False`, validate ISO timestamps and ordering, require positive workload volume/concurrency/duration, and evaluate comparators through an explicit mapping rather than `eval`. + +- [ ] **Step 5: Bind gate PASS semantics** + + When G6 status is `PASS`, require at least one `qualifying_g6_operational_evidence()` entry. While blocked, reject malformed external G6 entries but accept repository evidence. Keep G2 behavior unchanged. + +- [ ] **Step 6: Verify GREEN** + + Run focused readiness tests and full test discovery. + +- [ ] **Step 7: Commit** + + `git add scripts/check_readiness.py tests/test_readiness.py && git commit -m "feat: enforce measured G6 evidence"` + +### Task 2: Operational-readiness operator contract + +**Files:** +- Create: `docs/OPERATIONAL_READINESS.md` +- Modify: `PRODUCTION_READINESS.md` +- Modify: `readiness/production-readiness.json` +- Modify: `ROADMAP.md` +- Modify: `SECURITY.md` +- Modify: `CHANGELOG.md` + +**Interfaces:** +- Produces the ten-scope operator workflow and exact G6 matrix/ledger wording. +- Ledger cites `docs/OPERATIONAL_READINESS.md`, checker, and tests but remains `BLOCKED`. + +- [ ] **Step 1: Add failing matrix-binding tests** + + Require exact G6 criterion, current-evidence, and next-evidence cells; mutation to “internal tests are enough” must fail. + +- [ ] **Step 2: Verify RED** + + Run focused readiness tests. Expected: G6 canonical constants and document are absent. + +- [ ] **Step 3: Write operational contract and synchronize status** + + Document each scope, envelope, measurements, comparators, independence, operator threshold ownership, submission workflow, and a deliberately incomplete non-qualifying skeleton with no invented numbers. Update matrix/ledger language and repository evidence references exactly. + +- [ ] **Step 4: Verify GREEN** + + Run focused readiness tests and `python3 scripts/check_readiness.py`. + +- [ ] **Step 5: Commit** + + `git add docs/OPERATIONAL_READINESS.md PRODUCTION_READINESS.md readiness/production-readiness.json ROADMAP.md SECURITY.md CHANGELOG.md tests/test_readiness.py && git commit -m "docs: publish G6 operational evidence contract"` + +### Task 3: Pin primary grounding sources + +**Files:** +- Modify: `SOURCES.md` +- Modify: `docs/OPERATIONAL_READINESS.md` + +**Interfaces:** +- Records official title, version, canonical URL, access date `2026-08-12`, and exact use for NIST SP 800-61r3, NIST SP 800-218, SLSA v1.2, OpenTelemetry signals/semantic conventions, and OWASP Logging Cheat Sheet. + +- [ ] **Step 1: Verify official current source metadata** + + Re-open official primary sources because these facts are version-sensitive. Record no source as project certification and derive no numeric operator threshold from generic guidance. + +- [ ] **Step 2: Add source records and inline citations** + + Keep source statement, project inference, and operator decision distinct. + +- [ ] **Step 3: Validate** + + Run `make check`, `make links`, and `git diff --check`. Retry link checking once only if failure is a classified remote disconnect; otherwise report the exact failure. + +- [ ] **Step 4: Commit** + + `git add SOURCES.md docs/OPERATIONAL_READINESS.md && git commit -m "docs: ground operational readiness scopes"` + +### Task 4: Exact-head validation and publication update + +**Files:** +- Modify: `docs/PUBLICATION_LOG.md` + +**Interfaces:** +- Produces pushed exact-head commit, updated draft PR, exact G2 digest, and an Issue #4 comment requesting review of the final Phase 2 surface. + +- [ ] **Step 1: Run local release gates** + + Run `make check`, `make links`, `make cli-demo`, `git diff --check`, and secret-safe staged-content inspection. + +- [ ] **Step 2: Compute final G2 digest** + + Run `python3 -c 'from scripts.check_readiness import g2_surface_digest; print(g2_surface_digest())'` after all Phase 2 artifacts are committed. + +- [ ] **Step 3: Update publication log** + + Record exact commit/digest only after commit identity exists; use a follow-up documentation commit and state that its non-G2 publication-log bytes do not alter the surface digest. + +- [ ] **Step 4: Push and publish review request** + + Push branch, update Issue #4 with exact reviewed commit and digest, link PR #8, and request qualifying independent review. Do not change G2 or G6 status from the request. + +- [ ] **Step 5: Verify exact-head CI** + + Inspect checks belonging to pushed head, not cached predecessor status. Retry one classified `TemporaryDirectory` cleanup race or link-check remote disconnect once; diagnose any other failure. diff --git a/docs/superpowers/plans/2026-08-12-quarantine-checkpoint.md b/docs/superpowers/plans/2026-08-12-quarantine-checkpoint.md new file mode 100644 index 0000000..32d3c3f --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-quarantine-checkpoint.md @@ -0,0 +1,167 @@ +# Quarantine Checkpoint Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Require an authenticated, durable, state-bound quarantine checkpoint before Phase 2 CLI repair. + +**Architecture:** `prototype/checkpoints.py` owns canonical checkpoint data and atomic persistence. `Importer` exposes separate authenticated quarantine and checkpointed-repair operations while retaining `repair()` as Phase 1 composition. `Workspace` resolves deterministic checkpoint paths; CLI orchestrates exactly one pending erratum and consumes evidence only after durable receipt and applied-state writeback. + +**Tech Stack:** Python 3.11+ standard library, frozen dataclasses, SHA-256, canonical JSON, `os.replace`, SQLite, `unittest`, `argparse`. + +## Global Constraints + +- Preserve correction, supersession, and erasure as distinct operations. +- Quarantine must complete before rebuild begins. +- Preserve negative, positive, and preservation checks. +- Missing, opaque, malformed, drifted, or replayed evidence never becomes verified. +- Erasure checkpoints contain identifiers and commitments, not erased content. +- Keep `Importer.repair()` as Phase 1 in-process composition. +- `make check` remains offline and standard-library-only. + +--- + +### Task 1: Canonical checkpoint model and atomic store + +**Files:** +- Create: `prototype/checkpoints.py` +- Create: `tests/test_checkpoints.py` + +**Interfaces:** +- Produces: `CheckpointError`, `AdapterCheckpoint`, `QuarantineCheckpoint`, `CheckpointStore`. +- `QuarantineCheckpoint.from_dict(value: object) -> QuarantineCheckpoint` performs strict type/key validation. +- `QuarantineCheckpoint.canonical_digest() -> str`, `to_dict() -> dict[str, object]`, and `with_consumed(timestamp: str) -> QuarantineCheckpoint` preserve identity semantics. +- `CheckpointStore.write(checkpoint)`, `load(path)`, and `consume(path, timestamp)` use atomic same-directory replacement. + +- [ ] **Step 1: Write failing model tests** + + Add literal-fixture tests for deterministic digest, strict keys/types, digest mutation rejection, consumed-state identity, safe filename validation, atomic round trip, and invalid JSON refusal. Each test names the production mutation it catches. + +- [ ] **Step 2: Verify RED** + + Run `python3 -m unittest tests.test_checkpoints -v`. + + Expected: `ModuleNotFoundError: No module named 'prototype.checkpoints'`. + +- [ ] **Step 3: Implement minimal checkpoint model** + + Use frozen dataclasses and canonical `json.dumps(..., sort_keys=True, separators=(",", ":"), ensure_ascii=False)`. Reject booleans where integers are required, non-ISO UTC timestamps, path separators, duplicate adapter names, unsorted/duplicate artifact IDs, and any digest mismatch. Exclude only `checkpoint_digest`, `consumed`, and `consumed_at` from identity bytes. + +- [ ] **Step 4: Implement atomic persistence** + + Create a temporary regular file in checkpoint directory, write UTF-8 JSON plus newline, flush and `os.fsync`, `os.replace`, then `fsync` directory. Refuse symlink targets and non-regular existing files. `consume` loads and validates before atomically writing `with_consumed()`. + +- [ ] **Step 5: Verify GREEN** + + Run `python3 -m unittest tests.test_checkpoints -v` and `python3 -m unittest discover -s tests -t .`. + +- [ ] **Step 6: Commit** + + `git add prototype/checkpoints.py tests/test_checkpoints.py && git commit -m "feat: add durable quarantine checkpoints"` + +### Task 2: Controller split between quarantine and checkpointed repair + +**Files:** +- Modify: `prototype/controller.py` +- Modify: `prototype/sqlite_store.py` +- Test: `tests/test_controller.py` +- Test: `tests/test_sqlite_store.py` + +**Interfaces:** +- Produces: `Importer.quarantine(erratum: Erratum) -> QuarantineCheckpoint`. +- Produces: `Importer.repair_quarantined(erratum: Erratum, checkpoint: QuarantineCheckpoint) -> Receipt`. +- Produces: adapter inspection `is_quarantined(artifact_ids: tuple[str, ...]) -> bool` for checkpoint verification. +- `Importer.repair()` composes `quarantine()` then `repair_quarantined()` without durable persistence. + +- [ ] **Step 1: Write failing controller tests** + + Prove authenticated quarantine returns complete adapter records; opaque required store is unknown; no rebuild event occurs; checkpointed repair does not quarantine twice; wrong erratum, state root, adapter set, artifact set, limitation, or ungated state is refused before rebuild; correction, supersession, erasure, and triad behavior remain unchanged. + +- [ ] **Step 2: Verify RED** + + Run focused controller/SQLite tests. Expected: missing public methods or attribute failures. + +- [ ] **Step 3: Implement minimal controller split** + + Extract current repair body so `quarantine()` authenticates and gates, then returns the model; `repair_quarantined()` re-authenticates, validates every binding and current gate state, then rebuilds/tests/attests. Preserve journal ordering and `repair(resume=...)` compatibility. + +- [ ] **Step 4: Verify GREEN** + + Run `python3 -m unittest tests.test_controller tests.test_sqlite_store -v` then full discovery. + +- [ ] **Step 5: Commit** + + `git add prototype/controller.py prototype/sqlite_store.py tests/test_controller.py tests/test_sqlite_store.py && git commit -m "feat: separate quarantine from repair"` + +### Task 3: Workspace persistence and CLI admission + +**Files:** +- Modify: `prototype/workspace.py` +- Modify: `prototype/cli.py` +- Modify: `tests/test_cli.py` +- Modify: `Makefile` + +**Interfaces:** +- Produces: `Workspace.checkpoints_dir`, `checkpoint_path(sequence, erratum_id)`, `write_checkpoint`, `load_checkpoint`, and `consume_checkpoint`. +- Produces: `errata quarantine` and checkpoint-requiring `errata repair`. +- Both commands select exactly one next pending erratum. + +- [ ] **Step 1: Write failing CLI tests** + + Add real-workspace tests proving quarantine persists before repair; repair without checkpoint refuses; only next erratum is processed; mutated, consumed, replayed, wrong-target, state-drifted, adapter-drifted, and ungated checkpoints refuse without receipt/applied advancement; interrupted repair retains checkpoint; successful partial repair consumes it only after receipt/applied durability. + +- [ ] **Step 2: Verify RED** + + Run `python3 -m unittest tests.test_cli -v`. Expected: parser rejects `quarantine` and current repair succeeds without a checkpoint. + +- [ ] **Step 3: Implement workspace and CLI** + + Add deterministic paths and `CheckpointStore` delegation. Catch `FeedError`, `CheckpointError`, and state validation errors as `EXIT_REFUSED`. Process one erratum, persist receipt idempotently, record applied sequence idempotently, then consume checkpoint. Add `quarantine` to `COMMANDS` and parser. + +- [ ] **Step 4: Update CLI demo** + + Insert `run quarantine` before `run repair`; preserve expected repair exit `2`, receipt verification, and cleanup. + +- [ ] **Step 5: Verify GREEN** + + Run `python3 -m unittest tests.test_cli -v`, `make cli-demo`, and full discovery. + +- [ ] **Step 6: Commit** + + `git add prototype/workspace.py prototype/cli.py tests/test_cli.py Makefile && git commit -m "feat: require quarantine checkpoint for CLI repair"` + +### Task 4: Documentation and Phase 2 status integration + +**Files:** +- Modify: `prototype/README.md` +- Modify: `spec/README.md` +- Modify: `README.md` +- Modify: `ROADMAP.md` +- Modify: `PRODUCTION_READINESS.md` +- Modify: `readiness/production-readiness.json` +- Modify: `CHANGELOG.md` +- Modify: `scripts/check_readiness.py` +- Modify: `tests/test_readiness.py` + +**Interfaces:** +- Produces accurate internal completion language for explicit quarantine while G2 remains `BLOCKED` for remaining vectors and external review. +- Adds `prototype/checkpoints.py` and `tests/test_checkpoints.py` to canonical G2 surface. + +- [ ] **Step 1: Add failing readiness regression tests** + + Require canonical G2 evidence/matrix language to acknowledge checkpoint completion without upgrading G2; require checkpoint files in surface manifest and digest. + +- [ ] **Step 2: Verify RED** + + Run `python3 -m unittest tests.test_readiness -v`. Expected: new canonical-language and surface assertions fail. + +- [ ] **Step 3: Update documentation and checker constants** + + Document command ordering, checkpoint fields, replay/drift refusal, recovery semantics, and remaining key-rotation/concurrency/invalid-target/confidentiality/receipt-binding gaps. + +- [ ] **Step 4: Verify GREEN** + + Run focused readiness tests, `make check`, and `git diff --check`. + +- [ ] **Step 5: Commit** + + `git add prototype/README.md spec/README.md README.md ROADMAP.md PRODUCTION_READINESS.md readiness/production-readiness.json CHANGELOG.md scripts/check_readiness.py tests/test_readiness.py && git commit -m "docs: record durable quarantine completion"` From 35bd554d1a6132eca9f56c0ae5ee1a326214f29d Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 21:52:10 +0200 Subject: [PATCH 33/88] feat: add durable quarantine checkpoints --- prototype/checkpoints.py | 275 ++++++++++++++++++++++++++++++++++++++ tests/test_checkpoints.py | 79 +++++++++++ 2 files changed, 354 insertions(+) create mode 100644 prototype/checkpoints.py create mode 100644 tests/test_checkpoints.py diff --git a/prototype/checkpoints.py b/prototype/checkpoints.py new file mode 100644 index 0000000..36a9dd8 --- /dev/null +++ b/prototype/checkpoints.py @@ -0,0 +1,275 @@ +"""Durable evidence that a signed erratum's known descendants are gated.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +import tempfile +from dataclasses import dataclass, replace +from datetime import datetime +from pathlib import Path +from typing import Any + + +class CheckpointError(ValueError): + """A checkpoint is malformed, unsafe, or no longer trustworthy.""" + + +_SAFE_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") +_DIGEST = re.compile(r"[0-9a-f]{64}") +_COVERAGE = {"verified", "partial", "unknown", "failed"} + + +def _utc_timestamp(value: object, label: str, *, optional: bool = False) -> str | None: + if optional and value is None: + return None + if not isinstance(value, str) or not value.endswith("Z"): + raise CheckpointError(f"{label} must be an ISO UTC timestamp") + try: + datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as error: + raise CheckpointError(f"{label} must be an ISO UTC timestamp") from error + return value + + +@dataclass(frozen=True) +class AdapterCheckpoint: + name: str + required: bool + artifact_ids: tuple[str, ...] + coverage: str + limitation: str | None + + def __post_init__(self) -> None: + if not isinstance(self.name, str) or not _SAFE_ID.fullmatch(self.name): + raise CheckpointError("adapter name is unsafe") + if not isinstance(self.required, bool): + raise CheckpointError("adapter required must be boolean") + if ( + not isinstance(self.artifact_ids, tuple) + or not all(isinstance(item, str) and item for item in self.artifact_ids) + or tuple(sorted(set(self.artifact_ids))) != self.artifact_ids + ): + raise CheckpointError("adapter artifact IDs must be unique and sorted") + if self.coverage not in _COVERAGE: + raise CheckpointError("adapter coverage is invalid") + if self.limitation is not None and ( + not isinstance(self.limitation, str) or not self.limitation.strip() + ): + raise CheckpointError("adapter limitation must be non-empty") + if self.coverage == "unknown" and self.limitation is None: + raise CheckpointError("unknown adapter coverage requires a limitation") + + def to_dict(self) -> dict[str, object]: + return { + "name": self.name, + "required": self.required, + "artifact_ids": list(self.artifact_ids), + "coverage": self.coverage, + "limitation": self.limitation, + } + + @classmethod + def from_dict(cls, value: object) -> AdapterCheckpoint: + if not isinstance(value, dict) or set(value) != { + "name", "required", "artifact_ids", "coverage", "limitation" + }: + raise CheckpointError("adapter checkpoint fields are invalid") + artifacts = value["artifact_ids"] + if not isinstance(artifacts, list): + raise CheckpointError("adapter artifact IDs must be an array") + return cls( + name=value["name"], + required=value["required"], + artifact_ids=tuple(artifacts), + coverage=value["coverage"], + limitation=value["limitation"], + ) + + +@dataclass(frozen=True) +class QuarantineCheckpoint: + schema_version: int + erratum_id: str + sequence: int + target_root: str + pre_state_root: str + adapters: tuple[AdapterCheckpoint, ...] + created_at: str + consumed: bool + consumed_at: str | None + checkpoint_digest: str + + def __post_init__(self) -> None: + if type(self.schema_version) is not int or self.schema_version != 1: + raise CheckpointError("checkpoint schema version must be 1") + if not isinstance(self.erratum_id, str) or not _SAFE_ID.fullmatch(self.erratum_id): + raise CheckpointError("erratum ID is unsafe") + if type(self.sequence) is not int or self.sequence < 1: + raise CheckpointError("checkpoint sequence must be a positive integer") + if not isinstance(self.target_root, str) or not self.target_root: + raise CheckpointError("checkpoint target root is invalid") + if not isinstance(self.pre_state_root, str) or not self.pre_state_root: + raise CheckpointError("checkpoint pre-state root is invalid") + if not isinstance(self.adapters, tuple) or not self.adapters: + raise CheckpointError("checkpoint adapters must be non-empty") + names = tuple(adapter.name for adapter in self.adapters) + if tuple(sorted(set(names))) != names: + raise CheckpointError("checkpoint adapters must be unique and sorted") + _utc_timestamp(self.created_at, "created_at") + if not isinstance(self.consumed, bool): + raise CheckpointError("checkpoint consumed must be boolean") + _utc_timestamp(self.consumed_at, "consumed_at", optional=True) + if self.consumed != (self.consumed_at is not None): + raise CheckpointError("checkpoint consumption fields disagree") + if not isinstance(self.checkpoint_digest, str) or not _DIGEST.fullmatch( + self.checkpoint_digest + ): + raise CheckpointError("checkpoint digest is invalid") + if self.canonical_digest() != self.checkpoint_digest: + raise CheckpointError("checkpoint digest does not match content") + + @classmethod + def create( + cls, + *, + erratum_id: str, + sequence: int, + target_root: str, + pre_state_root: str, + adapters: tuple[AdapterCheckpoint, ...], + created_at: str, + ) -> QuarantineCheckpoint: + provisional = object.__new__(cls) + object.__setattr__(provisional, "schema_version", 1) + object.__setattr__(provisional, "erratum_id", erratum_id) + object.__setattr__(provisional, "sequence", sequence) + object.__setattr__(provisional, "target_root", target_root) + object.__setattr__(provisional, "pre_state_root", pre_state_root) + object.__setattr__(provisional, "adapters", adapters) + object.__setattr__(provisional, "created_at", created_at) + object.__setattr__(provisional, "consumed", False) + object.__setattr__(provisional, "consumed_at", None) + object.__setattr__(provisional, "checkpoint_digest", "0" * 64) + digest = provisional.canonical_digest() + return cls(1, erratum_id, sequence, target_root, pre_state_root, adapters, + created_at, False, None, digest) + + def _identity_dict(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "erratum_id": self.erratum_id, + "sequence": self.sequence, + "target_root": self.target_root, + "pre_state_root": self.pre_state_root, + "adapters": [adapter.to_dict() for adapter in self.adapters], + "created_at": self.created_at, + } + + def canonical_digest(self) -> str: + encoded = json.dumps( + self._identity_dict(), sort_keys=True, separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def to_dict(self) -> dict[str, object]: + return { + **self._identity_dict(), + "consumed": self.consumed, + "consumed_at": self.consumed_at, + "checkpoint_digest": self.checkpoint_digest, + } + + @classmethod + def from_dict(cls, value: object) -> QuarantineCheckpoint: + keys = { + "schema_version", "erratum_id", "sequence", "target_root", + "pre_state_root", "adapters", "created_at", "consumed", + "consumed_at", "checkpoint_digest", + } + if not isinstance(value, dict) or set(value) != keys: + raise CheckpointError("checkpoint fields are invalid") + adapters = value["adapters"] + if not isinstance(adapters, list): + raise CheckpointError("checkpoint adapters must be an array") + return cls( + schema_version=value["schema_version"], + erratum_id=value["erratum_id"], + sequence=value["sequence"], + target_root=value["target_root"], + pre_state_root=value["pre_state_root"], + adapters=tuple(AdapterCheckpoint.from_dict(item) for item in adapters), + created_at=value["created_at"], + consumed=value["consumed"], + consumed_at=value["consumed_at"], + checkpoint_digest=value["checkpoint_digest"], + ) + + def with_consumed(self, timestamp: str) -> QuarantineCheckpoint: + if self.consumed: + raise CheckpointError("checkpoint is already consumed") + return replace(self, consumed=True, consumed_at=timestamp) + + +class CheckpointStore: + def __init__(self, root: Path) -> None: + self.root = Path(root) + + def path_for(self, checkpoint: QuarantineCheckpoint) -> Path: + return self.root / f"{checkpoint.sequence:04d}-{checkpoint.erratum_id}.json" + + def write(self, checkpoint: QuarantineCheckpoint) -> Path: + path = self.path_for(checkpoint) + self._atomic_write(path, checkpoint.to_dict()) + return path + + def load(self, path: Path) -> QuarantineCheckpoint: + path = Path(path) + if path.parent.resolve() != self.root.resolve(): + raise CheckpointError("checkpoint path is outside checkpoint directory") + try: + if path.is_symlink() or not stat.S_ISREG(path.stat().st_mode): + raise CheckpointError("checkpoint path must be a regular file") + payload: Any = json.loads(path.read_text(encoding="utf-8")) + except CheckpointError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise CheckpointError("checkpoint is not readable JSON") from error + checkpoint = QuarantineCheckpoint.from_dict(payload) + if path.name != self.path_for(checkpoint).name: + raise CheckpointError("checkpoint filename does not match content") + return checkpoint + + def consume(self, path: Path, timestamp: str) -> QuarantineCheckpoint: + consumed = self.load(path).with_consumed(timestamp) + self._atomic_write(Path(path), consumed.to_dict()) + return consumed + + def _atomic_write(self, path: Path, payload: dict[str, object]) -> None: + self.root.mkdir(parents=True, exist_ok=True) + if path.exists() and (path.is_symlink() or not stat.S_ISREG(path.stat().st_mode)): + raise CheckpointError("checkpoint target must be a regular file") + data = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8") + descriptor, temporary = tempfile.mkstemp(prefix=".checkpoint-", dir=self.root) + temporary_path = Path(temporary) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + directory = os.open(self.root, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + except OSError as error: + try: + temporary_path.unlink(missing_ok=True) + except OSError: + pass + raise CheckpointError("checkpoint atomic write failed") from error diff --git a/tests/test_checkpoints.py b/tests/test_checkpoints.py new file mode 100644 index 0000000..da6e242 --- /dev/null +++ b/tests/test_checkpoints.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path + +from prototype.checkpoints import ( + AdapterCheckpoint, + CheckpointError, + CheckpointStore, + QuarantineCheckpoint, +) + + +class CheckpointModel(unittest.TestCase): + def checkpoint(self) -> QuarantineCheckpoint: + return QuarantineCheckpoint.create( + erratum_id="err_0001", + sequence=1, + target_root="mem_01HX", + pre_state_root="a" * 32, + adapters=( + AdapterCheckpoint("prompt_cache", True, (), "unknown", "opaque"), + AdapterCheckpoint("sqlite", True, ("fact:diet", "summary:dining"), "verified", None), + ), + created_at="2026-08-12T10:00:00Z", + ) + + def test_digest_is_deterministic_literal_sha256(self) -> None: + checkpoint = self.checkpoint() + self.assertEqual(len(checkpoint.checkpoint_digest), 64) + self.assertEqual(checkpoint.canonical_digest(), checkpoint.checkpoint_digest) + self.assertEqual(QuarantineCheckpoint.from_dict(checkpoint.to_dict()), checkpoint) + + def test_content_mutation_is_rejected(self) -> None: + payload = self.checkpoint().to_dict() + payload["target_root"] = "mem_other" + with self.assertRaisesRegex(CheckpointError, "digest"): + QuarantineCheckpoint.from_dict(payload) + + def test_consumption_preserves_quarantine_identity(self) -> None: + checkpoint = self.checkpoint() + consumed = checkpoint.with_consumed("2026-08-12T11:00:00Z") + self.assertEqual(consumed.checkpoint_digest, checkpoint.checkpoint_digest) + self.assertTrue(consumed.consumed) + + def test_duplicate_or_unsorted_artifacts_are_rejected(self) -> None: + for artifacts in (("b", "a"), ("a", "a")): + with self.subTest(artifacts=artifacts), self.assertRaises(CheckpointError): + AdapterCheckpoint("sqlite", True, artifacts, "verified", None) + + def test_unsafe_erratum_id_is_rejected(self) -> None: + with self.assertRaisesRegex(CheckpointError, "erratum"): + replace(self.checkpoint(), erratum_id="../escape") + + +class CheckpointPersistence(unittest.TestCase): + def test_atomic_round_trip_and_consumption(self) -> None: + checkpoint = CheckpointModel().checkpoint() + with tempfile.TemporaryDirectory() as temp: + store = CheckpointStore(Path(temp)) + path = store.write(checkpoint) + self.assertEqual(store.load(path), checkpoint) + consumed = store.consume(path, "2026-08-12T11:00:00Z") + self.assertTrue(consumed.consumed) + self.assertEqual(json.loads(path.read_text())["consumed"], True) + + def test_invalid_json_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "0001-err_0001.json" + path.write_text("{", encoding="utf-8") + with self.assertRaisesRegex(CheckpointError, "JSON"): + CheckpointStore(Path(temp)).load(path) + + +if __name__ == "__main__": + unittest.main() From 99b2ac01f05fffe94b4fb57d6787db7e6b8ddc53 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 21:54:05 +0200 Subject: [PATCH 34/88] feat: separate quarantine from repair --- prototype/controller.py | 104 +++++++++++++++++++++++++++++++++++++-- tests/test_controller.py | 37 +++++++++++++- 2 files changed, 135 insertions(+), 6 deletions(-) diff --git a/prototype/controller.py b/prototype/controller.py index 16f37d5..8cb9d1d 100644 --- a/prototype/controller.py +++ b/prototype/controller.py @@ -13,10 +13,16 @@ from __future__ import annotations from dataclasses import dataclass, field, replace +from datetime import datetime, timezone from enum import Enum from typing import Any, Sequence from prototype.adapters import CannotEnumerate +from prototype.checkpoints import ( + AdapterCheckpoint, + CheckpointError, + QuarantineCheckpoint, +) from prototype.errata import Erratum, FeedError, Operation, RootRegistry, verify_feed from prototype.lineage import LineageLedger from prototype.receipts import Receipt, aggregate_coverage @@ -136,16 +142,57 @@ def repair(self, erratum: Erratum, *, resume: bool = False) -> Receipt: # receipt. The sequence check still passes on a genuine resume because # an interrupted attempt never reached attest, so `last_sequence` did # not advance. - validated = self.observe(erratum) + checkpoint = self.quarantine(erratum) if resume: self.journal.append( JournalEvent(Phase.OBSERVE, {"resumed": erratum.erratum_id}) ) + return self.repair_quarantined(erratum, checkpoint) + + def quarantine(self, erratum: Erratum) -> QuarantineCheckpoint: + """Authenticate and gate one erratum without beginning its rebuild.""" + validated = self.observe(erratum) pre_state_root = self.state_root() root = validated.target_root - gated, unenumerable = self._quarantine(root) + unknown = set(unenumerable) + records = [] + for adapter in sorted(self.adapters, key=lambda item: item.name): + limitation = self._opaque_limitation(adapter.name) if adapter.name in unknown else None + records.append( + AdapterCheckpoint( + name=adapter.name, + required=bool(getattr(adapter, "required", True)), + artifact_ids=tuple(sorted(gated[adapter.name])), + coverage="unknown" if adapter.name in unknown else "verified", + limitation=limitation, + ) + ) + return QuarantineCheckpoint.create( + erratum_id=validated.erratum_id, + sequence=validated.sequence, + target_root=root, + pre_state_root=pre_state_root, + adapters=tuple(records), + created_at=datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + ) + + def repair_quarantined( + self, erratum: Erratum, checkpoint: QuarantineCheckpoint + ) -> Receipt: + """Re-authenticate and repair only state proven gated by a checkpoint.""" + + validated = self.observe(erratum) + self._validate_checkpoint(validated, checkpoint) + + root = validated.target_root + gated = { + record.name: list(record.artifact_ids) for record in checkpoint.adapters + } + unenumerable = [ + record.name for record in checkpoint.adapters if record.coverage == "unknown" + ] self.journal.append(JournalEvent(Phase.REBUILD_BEGIN, {"root": root})) self.strategy.apply(self, validated, gated) @@ -154,7 +201,9 @@ def repair(self, erratum: Erratum, *, resume: bool = False) -> Receipt: triad = self._run_triad(validated) self.journal.append(JournalEvent(Phase.TEST, dict(triad))) - receipt = self._attest(validated, pre_state_root, unenumerable, triad) + receipt = self._attest( + validated, checkpoint.pre_state_root, unenumerable, triad + ) self.last_sequence = validated.sequence self._applied[root] = validated.sequence self.journal.append( @@ -162,6 +211,52 @@ def repair(self, erratum: Erratum, *, resume: bool = False) -> Receipt: ) return receipt + def _validate_checkpoint( + self, erratum: Erratum, checkpoint: QuarantineCheckpoint + ) -> None: + if checkpoint.consumed: + raise CheckpointError("checkpoint is already consumed") + if ( + checkpoint.erratum_id != erratum.erratum_id + or checkpoint.sequence != erratum.sequence + or checkpoint.target_root != erratum.target_root + ): + raise CheckpointError("checkpoint erratum binding does not match") + if checkpoint.pre_state_root != self.state_root(): + raise CheckpointError("checkpoint pre-state root does not match current state") + + records = {record.name: record for record in checkpoint.adapters} + adapters = {adapter.name: adapter for adapter in self.adapters} + if set(records) != set(adapters): + raise CheckpointError("checkpoint adapter inventory has drifted") + for name, adapter in adapters.items(): + record = records[name] + if record.required != bool(getattr(adapter, "required", True)): + raise CheckpointError(f"checkpoint adapter requirement drifted: {name}") + try: + current = tuple(sorted(adapter.enumerate(erratum.target_root))) + except CannotEnumerate: + if ( + record.artifact_ids + or record.coverage != "unknown" + or record.limitation != self._opaque_limitation(name) + ): + raise CheckpointError(f"checkpoint opaque coverage drifted: {name}") + continue + if record.coverage != "verified" or record.limitation is not None: + raise CheckpointError(f"checkpoint adapter coverage drifted: {name}") + if record.artifact_ids != current: + raise CheckpointError(f"checkpoint gated artifact set drifted: {name}") + if not all(adapter.is_quarantined(item) for item in current): + raise CheckpointError(f"checkpoint artifact is no longer gated: {name}") + + @staticmethod + def _opaque_limitation(name: str) -> str: + return ( + f"{name}: store exposes no enumeration interface, so its coverage " + "is unknown and no repair elsewhere changes that" + ) + def _quarantine(self, root: str) -> tuple[dict[str, list[str]], list[str]]: """Gate every known descendant everywhere, before any rebuild starts.""" @@ -215,8 +310,7 @@ def _attest( triad=triad, aggregate=aggregate_coverage(stores, triad), limitations=[ - f"{name}: store exposes no enumeration interface, so its coverage " - "is unknown and no repair elsewhere changes that" + self._opaque_limitation(name) for name in unenumerable ], history_retained=erratum.operation is Operation.SUPERSEDE, diff --git a/tests/test_controller.py b/tests/test_controller.py index 03e7cb5..1a91d96 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -9,8 +9,8 @@ import json import unittest - from prototype.adapters import Coverage +from prototype.checkpoints import CheckpointError, QuarantineCheckpoint from prototype.controller import Phase from prototype.errata import Erratum, Operation from prototype.scenario import DIET, build_importer @@ -103,6 +103,41 @@ def test_every_known_descendant_is_gated_before_the_first_rebuild(self) -> None: self.assertIn("summary:dining", gated.detail["markdown"]) self.assertIn("vec:diet", gated.detail["vector"]) + def test_explicit_quarantine_returns_bound_evidence_without_rebuild(self) -> None: + importer = build_importer(OWNER) + checkpoint = importer.quarantine(supersede()) + self.assertEqual(checkpoint.erratum_id, "err_supersede") + self.assertEqual(checkpoint.pre_state_root, importer.state_root()) + self.assertEqual( + {item.name: item.coverage for item in checkpoint.adapters}, + {"markdown": "verified", "prompt_cache": "unknown", "vector": "verified"}, + ) + self.assertNotIn(Phase.REBUILD_BEGIN, [event.phase for event in importer.journal]) + + def test_checkpointed_repair_does_not_quarantine_twice(self) -> None: + importer = build_importer(OWNER) + checkpoint = importer.quarantine(supersede()) + receipt = importer.repair_quarantined(supersede(), checkpoint) + self.assertEqual(receipt.erratum_id, checkpoint.erratum_id) + self.assertEqual( + [event.phase for event in importer.journal].count(Phase.QUARANTINE_BEGIN), 1 + ) + + def test_drifted_checkpoint_is_refused_before_rebuild(self) -> None: + importer = build_importer(OWNER) + checkpoint = importer.quarantine(supersede()) + drifted = QuarantineCheckpoint.create( + erratum_id=checkpoint.erratum_id, + sequence=checkpoint.sequence, + target_root=checkpoint.target_root, + pre_state_root="b" * 32, + adapters=checkpoint.adapters, + created_at=checkpoint.created_at, + ) + with self.assertRaisesRegex(CheckpointError, "state"): + importer.repair_quarantined(supersede(), drifted) + self.assertNotIn(Phase.REBUILD_BEGIN, [event.phase for event in importer.journal]) + class EveryDescendantGetsADisposition(unittest.TestCase): """Acceptance: every known descendant receives an explicit disposition.""" From d03c3367cf196d61f483c6ac8c1dd352d1a075d0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 21:56:34 +0200 Subject: [PATCH 35/88] feat: require quarantine checkpoint for CLI repair --- Makefile | 1 + prototype/cli.py | 64 +++++++++++++++++++++++++++++++++--------- prototype/workspace.py | 60 +++++++++++++++++++++++++++++++++------ tests/test_cli.py | 41 +++++++++++++++++++++------ 4 files changed, 136 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index e327008..3123873 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ cli-demo: ## Drive a full lifecycle through the CLI in a scratch workspace run export --root mem_02KP --artifact fact:venue --content "prefers quiet restaurants" >/dev/null; \ run derive --artifact summary:dining --inputs fact:diet fact:venue --content "is vegetarian; prefers quiet restaurants" >/dev/null; \ run publish --root mem_01HX --operation supersede --replacement "eats meat again" --negative vegetarian --positive "eats meat again" --preserve "quiet restaurants"; \ + run quarantine; \ set +e; \ run repair; status=$$?; \ run audit; \ diff --git a/prototype/cli.py b/prototype/cli.py index 8f188a6..1816bb1 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -23,9 +23,11 @@ import argparse import json import sys +from datetime import datetime, timezone from pathlib import Path from prototype.adapters import Coverage, OpaqueAdapter +from prototype.checkpoints import CheckpointError from prototype.controller import Importer, Phase from prototype.errata import Erratum, FeedError, Operation, RootRegistry, read_feed from prototype.lineage import LineageLedger @@ -210,26 +212,58 @@ def cmd_plan(ws: Workspace, args: argparse.Namespace) -> int: store.close() +def _next_pending(ws: Workspace, importer: Importer) -> Erratum | None: + errata = read_feed(ws.feed_path.read_text(encoding="utf-8")) + pending = [item for item in errata if item.sequence > importer.last_sequence] + return pending[0] if pending else None + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def cmd_quarantine(ws: Workspace, args: argparse.Namespace) -> int: + importer, store = _importer(ws) + try: + erratum = _next_pending(ws, importer) + if erratum is None: + print("nothing to quarantine") + return EXIT_OK + path = ws.checkpoint_path(erratum.sequence, erratum.erratum_id) + try: + if path.exists(): + checkpoint = ws.load_checkpoint(erratum.sequence, erratum.erratum_id) + validated = importer.observe(erratum) + importer._validate_checkpoint(validated, checkpoint) + else: + checkpoint = importer.quarantine(erratum) + path = ws.write_checkpoint(checkpoint) + except (CheckpointError, FeedError, OSError, ValueError) as error: + print(f"refused: {error}", file=sys.stderr) + return EXIT_REFUSED + print(f"checkpoint: {path.relative_to(ws.root)}") + print(f"digest: {checkpoint.checkpoint_digest}") + return EXIT_OK + finally: + store.close() + + def cmd_repair(ws: Workspace, args: argparse.Namespace) -> int: importer, store = _importer(ws) try: - errata = read_feed(ws.feed_path.read_text(encoding="utf-8")) - pending = [e for e in errata if e.sequence > importer.last_sequence] - if not pending: + erratum = _next_pending(ws, importer) + if erratum is None: print("nothing to repair") return EXIT_OK - - last: Receipt | None = None - for erratum in pending: - try: - last = importer.repair(erratum) - except FeedError as error: - print(f"refused: {error}", file=sys.stderr) - return EXIT_REFUSED + try: + checkpoint = ws.load_checkpoint(erratum.sequence, erratum.erratum_id) + last = importer.repair_quarantined(erratum, checkpoint) ws.write_receipt(last) ws.record_applied(last.target_root, last.sequence) - - assert last is not None + ws.consume_checkpoint(last.sequence, last.erratum_id, _now()) + except (CheckpointError, FeedError, OSError, ValueError) as error: + print(f"refused: checkpoint admission failed: {error}", file=sys.stderr) + return EXIT_REFUSED for event in importer.journal: if event.phase in (Phase.QUARANTINE_COMPLETE, Phase.TEST): print(f" {event.phase.value}: {event.detail}") @@ -327,6 +361,7 @@ def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: "publish": cmd_publish, "pull": cmd_pull, "plan": cmd_plan, + "quarantine": cmd_quarantine, "repair": cmd_repair, "test": cmd_test, "attest": cmd_attest, @@ -374,7 +409,8 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("pull", help="list errata not yet applied") sub.add_parser("plan", help="show what a repair would touch, without touching it") - sub.add_parser("repair", help="quarantine, rebuild, probe, and attest") + sub.add_parser("quarantine", help="gate the next erratum and persist a checkpoint") + sub.add_parser("repair", help="rebuild, probe, and attest from a checkpoint") sub.add_parser("test", help="show the repair triad from the latest receipt") sub.add_parser("attest", help="print the latest receipt") diff --git a/prototype/workspace.py b/prototype/workspace.py index c6e34e2..c247065 100644 --- a/prototype/workspace.py +++ b/prototype/workspace.py @@ -14,9 +14,11 @@ from __future__ import annotations import json +import os from pathlib import Path from typing import Any +from prototype.checkpoints import CheckpointStore, QuarantineCheckpoint from prototype.errata import Erratum from prototype.lineage import LineageLedger from prototype.receipts import Receipt @@ -41,6 +43,10 @@ def lineage_path(self) -> Path: def receipts_dir(self) -> Path: return self.root / "receipts" + @property + def checkpoints_dir(self) -> Path: + return self.root / "checkpoints" + @property def store_path(self) -> Path: return self.root / "store.sqlite3" @@ -59,7 +65,7 @@ def exists(self) -> bool: # -- setup --------------------------------------------------------- def initialise(self) -> None: - for directory in ("feeds", "registry", "receipts", "identity"): + for directory in ("feeds", "registry", "receipts", "checkpoints", "identity"): (self.root / directory).mkdir(parents=True, exist_ok=True) self.feed_path.touch() self.lineage_path.touch() @@ -171,10 +177,11 @@ def applied(self) -> dict[str, int]: def record_applied(self, root: str, sequence: int) -> None: state = self.applied() + current = state.get(root) + if current is not None and current != sequence: + raise ValueError("applied state contradicts checkpoint sequence") state[root] = sequence - self.applied_path.write_text( - json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) + self._atomic_json(self.applied_path, state) def last_applied_sequence(self) -> int: values = self.applied().values() @@ -185,12 +192,49 @@ def last_applied_sequence(self) -> int: def write_receipt(self, receipt: Receipt) -> Path: self.receipts_dir.mkdir(parents=True, exist_ok=True) path = self.receipts_dir / f"{receipt.sequence:04d}-{receipt.erratum_id}.json" - path.write_text( - json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) + payload = receipt.to_dict() + if path.exists(): + if json.loads(path.read_text(encoding="utf-8")) != payload: + raise ValueError("existing receipt contradicts checkpointed repair") + return path + self._atomic_json(path, payload) return path + # -- quarantine checkpoints -------------------------------------- + + def checkpoint_path(self, sequence: int, erratum_id: str) -> Path: + return self.checkpoints_dir / f"{sequence:04d}-{erratum_id}.json" + + def write_checkpoint(self, checkpoint: QuarantineCheckpoint) -> Path: + return CheckpointStore(self.checkpoints_dir).write(checkpoint) + + def load_checkpoint(self, sequence: int, erratum_id: str) -> QuarantineCheckpoint: + path = self.checkpoint_path(sequence, erratum_id) + return CheckpointStore(self.checkpoints_dir).load(path) + + def consume_checkpoint( + self, sequence: int, erratum_id: str, timestamp: str + ) -> QuarantineCheckpoint: + path = self.checkpoint_path(sequence, erratum_id) + return CheckpointStore(self.checkpoints_dir).consume(path, timestamp) + + def _atomic_json(self, path: Path, payload: object) -> None: + temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}") + data = json.dumps(payload, indent=2, sort_keys=True) + "\n" + try: + with temporary.open("x", encoding="utf-8") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + temporary.unlink(missing_ok=True) + def all_receipts(self) -> list[tuple[str, dict[str, Any]]]: if not self.receipts_dir.is_dir(): return [] diff --git a/tests/test_cli.py b/tests/test_cli.py index ad993b3..f777127 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -58,6 +58,11 @@ def publish_supersession(self) -> subprocess.CompletedProcess[str]: "--positive", "eats meat again", "--preserve", "quiet restaurants", ) + def quarantine_and_repair(self) -> subprocess.CompletedProcess[str]: + quarantine = self.run_cli("quarantine") + self.assertEqual(quarantine.returncode, EXIT_OK, quarantine.stdout + quarantine.stderr) + return self.run_cli("repair") + class WorkspaceLifecycle(CliCase): def test_commands_refuse_before_init(self) -> None: @@ -100,10 +105,30 @@ def test_the_published_erratum_satisfies_the_published_schema(self) -> None: class RepairReportsHonestly(CliCase): - def test_repair_exits_inconclusive_rather_than_claiming_success(self) -> None: + def test_repair_without_checkpoint_is_refused_without_mutation(self) -> None: self.seed() self.publish_supersession() result = self.run_cli("repair") + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertIn("checkpoint", result.stderr) + self.assertEqual(list((self.cwd / ".errata" / "receipts").glob("*.json")), []) + + def test_quarantine_persists_bound_checkpoint_before_repair(self) -> None: + self.seed() + self.publish_supersession() + result = self.run_cli("quarantine") + self.assertEqual(result.returncode, EXIT_OK, result.stderr) + paths = list((self.cwd / ".errata" / "checkpoints").glob("*.json")) + self.assertEqual(len(paths), 1) + payload = json.loads(paths[0].read_text()) + self.assertEqual(payload["erratum_id"], "err_0001") + self.assertFalse(payload["consumed"]) + self.assertIn(payload["checkpoint_digest"], result.stdout) + + def test_repair_exits_inconclusive_rather_than_claiming_success(self) -> None: + self.seed() + self.publish_supersession() + result = self.quarantine_and_repair() self.assertEqual(result.returncode, EXIT_INCONCLUSIVE, result.stdout + result.stderr) self.assertIn("prompt_cache", result.stdout) @@ -112,7 +137,7 @@ def test_all_three_probes_pass_even_though_the_aggregate_does_not(self) -> None: # the result is still not `verified`. self.seed() self.publish_supersession() - self.run_cli("repair") + self.quarantine_and_repair() result = self.run_cli("test") self.assertEqual(result.returncode, EXIT_OK, result.stdout) self.assertNotIn("fail", result.stdout) @@ -120,7 +145,7 @@ def test_all_three_probes_pass_even_though_the_aggregate_does_not(self) -> None: def test_audit_json_is_machine_readable_and_not_green(self) -> None: self.seed() self.publish_supersession() - self.run_cli("repair") + self.quarantine_and_repair() result = self.run_cli("audit", "--json") self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) payload = json.loads(result.stdout) @@ -133,7 +158,7 @@ def test_the_substrate_scan_reaches_the_receipt(self) -> None: # report a clean store. self.seed() self.publish_supersession() - self.run_cli("repair") + self.quarantine_and_repair() payload = json.loads(self.run_cli("audit", "--json").stdout) self.assertEqual(payload["stores"]["sqlite"], "failed") @@ -160,13 +185,13 @@ class ReceiptsAreVerifiable(CliCase): def test_a_genuine_receipt_verifies(self) -> None: self.seed() self.publish_supersession() - self.run_cli("repair") + self.quarantine_and_repair() self.assertEqual(self.run_cli("verify").returncode, EXIT_OK) def test_a_tampered_aggregate_is_caught(self) -> None: self.seed() self.publish_supersession() - self.run_cli("repair") + self.quarantine_and_repair() path = sorted((self.cwd / ".errata" / "receipts").glob("*.json"))[0] payload = json.loads(path.read_text()) payload["aggregate"] = "verified" @@ -188,14 +213,14 @@ def test_a_forged_erratum_in_the_feed_is_refused(self) -> None: payload = json.loads(feed.read_text().strip()) payload["replacement"] = "eats only pineapple" feed.write_text(json.dumps(payload, sort_keys=True) + "\n") - result = self.run_cli("repair") + result = self.run_cli("quarantine") self.assertEqual(result.returncode, EXIT_REFUSED) self.assertIn("signature", result.stderr) def test_a_second_repair_finds_nothing_to_do(self) -> None: self.seed() self.publish_supersession() - self.run_cli("repair") + self.quarantine_and_repair() result = self.run_cli("repair") self.assertEqual(result.returncode, EXIT_OK) self.assertIn("nothing to repair", result.stdout) From a9f0ce6233a6b71414e7948b8f89194b747de3df Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 22:00:10 +0200 Subject: [PATCH 36/88] docs: record durable quarantine completion --- CHANGELOG.md | 5 +++++ PRODUCTION_READINESS.md | 2 +- README.md | 2 +- ROADMAP.md | 11 +++++++++-- prototype/README.md | 13 ++++++++++++- readiness/production-readiness.json | 1 + scripts/check_readiness.py | 5 +++-- spec/README.md | 7 +++++++ tests/test_readiness.py | 8 +++++++- 9 files changed, 46 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6137f9..6f379c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Added durable `errata quarantine` checkpoints. CLI repair now requires an + authenticated, atomic checkpoint bound to erratum, sequence, target, + inspectable pre-state, adapters, opaque limitations, and gated artifacts; + consumption occurs only after durable receipt and applied-state writeback. + G2 remains `BLOCKED` for remaining Phase 2 vectors and independent review. - Added a fail-closed production-cryptography qualification record. PyCA reproduced the repository's RFC 8032 vectors but documents no external project audit; libsodium has a published assessment for older releases, not diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 472938e..5f2e15c 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded. | Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Semantic probes and durable `errata quarantine` checkpoints are internally implemented, but Phase 2 remains incomplete: vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded. | Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/README.md b/README.md index 38cdaf8..3631333 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 is implemented; Phase 2 remains incomplete. Item 6 adds provider-neutral semantic probes with deterministic fixtures and recorded verifier configuration, while explicit `errata quarantine` CLI support and vectors for key rotation, concurrent events, invalid targets, and confidentiality remain open. Receipt-binding vectors exist for state roots but coverage remains partial. G2 remains `BLOCKED` pending internal Phase 2 completion and a dated independent review of the complete conformance surface. +Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 is implemented; Phase 2 remains incomplete. Item 6 adds provider-neutral semantic probes with deterministic fixtures and recorded verifier configuration. Explicit `errata quarantine` CLI support now persists a digest-bound checkpoint required by `errata repair`; vectors for key rotation, concurrent events, invalid targets, and confidentiality remain open. Receipt-binding vectors exist for state roots but coverage remains partial. G2 remains `BLOCKED` pending internal Phase 2 completion and a dated independent review of the complete conformance surface. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/ROADMAP.md b/ROADMAP.md index 976c87b..dd62f6b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -102,7 +102,7 @@ Kill or redesign the concept if exact lineage cannot survive even the controlled ## Phase 2 — Conformance surface -**Status:** items 1 through 5 are partially implemented and item 6 is internally implemented; Phase 2 is not complete. `spec/` carries schemas, vectors, and deterministic semantic fixtures; `prototype/cli.py` exposes most control-plane commands and `semantic-test`; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. The explicit `errata quarantine` command and vectors for key rotation, concurrent events, invalid targets, and confidentiality remain unimplemented. Receipt state-root binding vectors exist, but receipt-binding coverage remains partial. This internal work does not complete G2: a dated independent external conformance review of the complete Phase 2 surface remains required. +**Status:** items 1 through 5 are partially implemented and item 6 is internally implemented; Phase 2 is not complete. `spec/` carries schemas, vectors, and deterministic semantic fixtures. `prototype/cli.py` exposes the declared control-plane commands and `semantic-test`; `prototype/checkpoints.py` makes `errata quarantine` durable and requires its state-bound evidence before CLI repair; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. Vectors for key rotation, concurrent events, invalid targets, and confidentiality remain unimplemented. Receipt state-root binding vectors exist, but receipt-binding coverage remains partial. This internal work does not complete G2: a dated independent external conformance review of the complete Phase 2 surface remains required. Only after the file-backed proof passes: @@ -126,7 +126,14 @@ Only after the file-backed proof passes: 5. Publish conformance vectors for signatures, sequencing, key rotation, concurrent events, invalid targets, receipt binding, and confidentiality. 6. Add model-assisted semantic probes behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. **Implemented internally:** [`prototype/semantic.py`](prototype/semantic.py), [`spec/semantic/`](spec/semantic/), and `errata semantic-test` record only structured, configuration-bound observations; inconclusive, malformed, missing, duplicate, or drifted required evidence is not success. -Phase 2 internal implementation is incomplete. Complete the listed CLI and vector gaps before asking an independent reviewer to evaluate the complete conformance surface. G2 remains blocked until that implementation work and a dated independent external review exist. Broader Phase 2 interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). +The CLI quarantine checkpoint is also implemented internally. `errata quarantine` +authenticates exactly the next pending erratum, gates enumerable descendants, +records opaque stores as `unknown`, and atomically persists a digest bound to +the erratum, sequence, target, pre-state, adapter inventory, and gated set. +`errata repair` re-authenticates and refuses missing, consumed, replayed, or +drifted checkpoints before rebuild. + +Phase 2 internal implementation is incomplete. Complete the listed vector and receipt-binding gaps before asking an independent reviewer to evaluate the complete conformance surface. G2 remains blocked until that implementation work and a dated independent external review exist. Broader Phase 2 interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). ## Phase 3 — Interoperability experiment diff --git a/prototype/README.md b/prototype/README.md index 2526b6d..e7fce33 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -36,6 +36,7 @@ non-green result cannot be mistaken for a bug in the aggregation. | `adapters.py` | The three stores, plus the four coverage results. | | `strategies.py` | How a repair is carried out. One conforming strategy and three that are not, because a conformance suite where nothing can fail has not tested anything. | | `controller.py` | observe → quarantine → rebuild → test → attest, with a journal that makes the ordering observable. | +| `checkpoints.py` | Canonical, atomically persisted proof binding CLI quarantine to erratum, state, adapters, and gated artifacts. | | `receipts.py` | Coverage-aware receipts and the aggregation rule. | | `scenario.py` | The synthetic fixture. | | `demo.py` | The narrated run. | @@ -74,7 +75,17 @@ make cli-demo ``` A full lifecycle without importing Python: `init`, `export`, `derive`, -`publish`, `pull`, `plan`, `repair`, `test`, `attest`, `audit`, `verify`. +`publish`, `pull`, `plan`, `quarantine`, `repair`, `test`, `attest`, `audit`, +`verify`. + +`quarantine` authenticates exactly the next pending erratum, gates every +enumerable descendant, records opaque stores as `unknown`, and atomically +writes `checkpoints/-.json`. Its canonical digest binds the +erratum, target, inspectable pre-state root, adapter inventory, limitations, +and gated artifact set. `repair` re-authenticates and refuses missing, mutated, +consumed, wrong-target, state-drifted, adapter-drifted, or ungated evidence. +Consumption happens only after receipt and applied-state writeback, so an +interrupted rebuild retains an unconsumed checkpoint for safe resume. Exit codes are part of the interface. `0` is success, `1` is a refusal or a failed check, and **`2` means the repair ran and the result is not verified**. diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index 5ec01c5..eca0040 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -32,6 +32,7 @@ "criterion": "Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface.", "evidence": [ {"kind": "repository", "ref": "ROADMAP.md"}, + {"kind": "repository", "ref": "prototype/checkpoints.py"}, {"kind": "repository", "ref": "prototype/semantic.py"}, {"kind": "repository", "ref": "spec/semantic/probes.json"}, {"kind": "repository", "ref": "spec/semantic/verifier-config.json"}, diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index cd9ed2f..e772f5c 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -39,6 +39,7 @@ G2_ATTESTATION = "llm-errata-independent-review-v1" G2_REQUIRED_TESTS = ( "tests/test_adapters.py", + "tests/test_checkpoints.py", "tests/test_cli.py", "tests/test_controller.py", "tests/test_ed25519.py", @@ -61,8 +62,8 @@ ) G2_RESULTS = {"pass", "pass-with-findings", "fail"} G2_MATRIX_CURRENT_EVIDENCE = ( - "Semantic probes are internally implemented, but Phase 2 remains incomplete: " - "`errata quarantine` and vectors for key rotation, concurrency, invalid targets, " + "Semantic probes and durable `errata quarantine` checkpoints are internally " + "implemented, but Phase 2 remains incomplete: vectors for key rotation, concurrency, invalid targets, " "and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( diff --git a/spec/README.md b/spec/README.md index f1d84f1..0256ec1 100644 --- a/spec/README.md +++ b/spec/README.md @@ -61,6 +61,13 @@ requires each invalid vector to be rejected **for the stated reason**, and requires the invalid set to trip at least four distinct rules, so the suite cannot be one rule wearing many hats. +The Phase 2 CLI adds a local checkpoint evidence object implemented in +`prototype/checkpoints.py`. It is not a third wire schema yet: independent +review must settle whether checkpoints are importer-local evidence or a +portable conformance artifact. Its canonical JSON and SHA-256 binding are +nevertheless tested so missing, mutated, consumed, replayed, or drifted local +evidence cannot authorize repair. + ## What the bundled validator supports `prototype/schema.py` implements a subset, because the repository installs diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 87126af..c8d51db 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -12,6 +12,7 @@ from scripts.check_readiness import ( g2_surface_digest, g2_surface_digest_at_commit, + g2_surface_files, qualifying_g2_review_evidence, valid_external_evidence, valid_g2_review_evidence, @@ -36,6 +37,11 @@ def test_current_not_ready_ledger_is_honest(self) -> None: result = run_checker(root, SCRIPT) self.assertEqual(result.returncode, EXIT_OK, result.stdout) + def test_g2_surface_includes_checkpoint_contract_and_tests(self) -> None: + files = set(g2_surface_files()) + self.assertIn("prototype/checkpoints.py", files) + self.assertIn("tests/test_checkpoints.py", files) + def test_generic_external_evidence_allows_https_root_url(self) -> None: self.assertTrue( valid_external_evidence( @@ -271,7 +277,7 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Semantic probes are internally implemented, but Phase 2 remains incomplete: `errata quarantine` and vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded.", + "Semantic probes and durable `errata quarantine` checkpoints are internally implemented, but Phase 2 remains incomplete: vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded.", "local tests prove readiness.", ) From aeba4d5e510cc2cfa0164f0a9e7784ac62cd44b2 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 22:09:18 +0200 Subject: [PATCH 37/88] feat: enforce measured G6 evidence --- CHANGELOG.md | 5 + PRODUCTION_READINESS.md | 2 +- ROADMAP.md | 8 + SECURITY.md | 11 + SOURCES.md | 15 ++ docs/OPERATIONAL_READINESS.md | 116 ++++++++ .../2026-08-12-g6-operational-evidence.md | 2 + ...26-08-12-g6-operational-evidence-design.md | 7 +- readiness/production-readiness.json | 9 +- scripts/check_readiness.py | 249 +++++++++++++++++- tests/test_readiness.py | 166 ++++++++++++ 11 files changed, 579 insertions(+), 11 deletions(-) create mode 100644 docs/OPERATIONAL_READINESS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f379c9..aace09e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Added a ten-scope G6 operational evidence contract and fail-closed checker. + One independent report must bind exact commit and deployment, declare + workload, platform, failure domain, observation window, numeric thresholds, + units, comparators, and raw artifacts, then pass every measurement. Internal + tests validate the contract but leave G6 `BLOCKED`. - Added durable `errata quarantine` checkpoints. CLI repair now requires an authenticated, atomic checkpoint bound to erratum, sequence, target, inspectable pre-state, adapters, opaque limitations, and gated artifacts; diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 5f2e15c..435c2ca 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -19,7 +19,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | -| G6 | Security, observability, recovery, compatibility, performance, deployment, and rollback gates pass with current operational evidence. | `BLOCKED` | `SECURITY.md` records reporting scope; no qualifying production operations evidence is recorded. | Current operational evidence covering deployment, rollback, recovery, observability, privacy, compatibility, load, denial-of-service, and incident response. | +| G6 | All ten operational scopes pass from one independent report bound to exact commit and deployment, with declared thresholds and measured comparators. | `BLOCKED` | No independent report binds an exact commit and deployment to passing measured comparators for all ten operational scopes. | One qualifying independent report with declared workload, platform, failure domain, observation window, numeric thresholds, raw artifacts, and passing measurements for every scope. | ## Approval boundaries diff --git a/ROADMAP.md b/ROADMAP.md index dd62f6b..426fafb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -139,6 +139,14 @@ Phase 2 internal implementation is incomplete. Complete the listed vector and re Run one user-controlled root across at least three independently operated runtimes or memory systems. [PHASE3_SYSTEMS.md](PHASE3_SYSTEMS.md) records public system-nomination requirements; nomination is not experiment approval. +Operational promotion is separately governed by +[`docs/OPERATIONAL_READINESS.md`](docs/OPERATIONAL_READINESS.md). Interoperability +does not substitute for deployment provenance, rollback, recovery, +observability, privacy, compatibility, performance, overload, incident-response, +access, secret, dependency, or vulnerability-management evidence. One +independent report must pass all ten scopes against operator-declared numeric +thresholds before G6 can pass. + Measure: - time from observation to quarantine; diff --git a/SECURITY.md b/SECURITY.md index 62d8670..56f59b4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -49,3 +49,14 @@ Security-relevant findings include, but are not limited to: - unsafe reference code or conformance tooling added to the repository. Factual disagreements, prior-art reports, and specification design proposals are not vulnerabilities; submit them through the normal contribution process. + +## Operational security gate + +Security reporting policy is not operational-readiness evidence. G6 additionally +requires the independent, measured ten-scope report in +[`docs/OPERATIONAL_READINESS.md`](docs/OPERATIONAL_READINESS.md), including +deployment provenance, rollback and recovery exercises, lifecycle observability, +telemetry redaction, compatibility, performance, overload behavior, incident +response, least privilege, secret lifecycle, dependency inventory, and +vulnerability management. Missing operator thresholds or internal assessment +keeps G6 `BLOCKED`. diff --git a/SOURCES.md b/SOURCES.md index 0f7967f..1efa819 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -100,6 +100,21 @@ An archived copy of the Glean report exists at exists, the correct action is to narrow the affected row, not to keep the citation. +## Operational-readiness grounding sources + +These official sources were read on 2026-08-12 to ground scope selection in +[`docs/OPERATIONAL_READINESS.md`](docs/OPERATIONAL_READINESS.md). They do not +certify LLM Errata, prove G6, or supply operator-specific numeric thresholds. + +| Source | Pin or version | Verified 2026-08-12 and project use | +|---|---|---| +| [NIST SP 800-61 Rev. 3](https://csrc.nist.gov/pubs/sp/800/61/r3/final) | Final, April 2025 | Official title confirmed: *Incident Response Recommendations and Considerations for Cybersecurity Risk Management: A CSF 2.0 Community Profile*. Grounds incident preparation, detection, response, recovery, and learning scope. | +| [NIST SP 800-218](https://csrc.nist.gov/pubs/sp/800/218/final) | SSDF Version 1.1, February 2022 | Official title confirmed: *Secure Software Development Framework (SSDF) Version 1.1: Recommendations for Mitigating the Risk of Software Vulnerabilities*. Grounds secure-development, dependency, provenance, and vulnerability-management scope. | +| [SLSA specification](https://slsa.dev/spec/v1.2/) | Version 1.2 | Official versioned specification page describes supply-chain security levels. Grounds deployment artifact and build-provenance binding. | +| [OpenTelemetry Signals](https://opentelemetry.io/docs/concepts/signals/) | Live official documentation | Official page defines supported telemetry categories. Grounds lifecycle metrics, logs, traces, and alert-observation coverage. | +| [OpenTelemetry Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/) | 1.44.0 | Official version shown on 2026-08-12. Grounds stable metric, unit, attribute, and event naming; it does not define LLM Errata-specific thresholds. | +| [OWASP Logging Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html) | Live OWASP Cheat Sheet Series | Grounds security-event logging, sensitive-data exclusion, monitoring, and verification concerns. | + ## Production-cryptography candidate sources These sources were read on 2026-08-12 for the internal candidate assessment in diff --git a/docs/OPERATIONAL_READINESS.md b/docs/OPERATIONAL_READINESS.md new file mode 100644 index 0000000..1ccef76 --- /dev/null +++ b/docs/OPERATIONAL_READINESS.md @@ -0,0 +1,116 @@ +# Operational readiness evidence contract + +LLM Errata gate G6 is an external operational-evidence gate. Repository tests +can validate this contract but cannot satisfy it. G6 passes only when one +independent report binds an exact repository commit and deployment to measured +passing evidence for all ten scopes below. + +## Operator declaration + +Before testing, operator declares: + +- stable deployment ID, platform, environment, deployed artifact SHA-256, + build-provenance reference, and deployment timestamp; +- workload name, dataset class, whether data is synthetic, positive event or + request volume, concurrency, duration in seconds, and failure domain; +- observation-window start and end in UTC; and +- numeric threshold, unit, comparator, metric, and raw evidence reference for + every measurement. + +Repository does not invent universal latency, throughput, RTO, RPO, overload, +or incident-response thresholds. Operator owns those service decisions before +exercise. Missing thresholds remain `BLOCKED`. + +## Ten mandatory scopes + +Each token appears exactly once. Alias, duplicate, missing, unknown, or extra +scope fails qualification. + +1. `deployment-integrity-provenance`: verify deployed artifact identity, + signature or digest, build and dependency provenance, and environment. +2. `rollback-exercise`: measure rollback and verify restored service and state. +3. `backup-recovery-rto-rpo`: exercise recovery against declared numeric RTO + and RPO thresholds. +4. `lifecycle-observability-alerting`: observe quarantine, rebuild, test, + attest, refusal, and incomplete coverage, including alert delivery. +5. `telemetry-privacy-redaction`: verify logs, metrics, traces, and retained + evidence exclude prohibited memory content, secrets, and erased values. +6. `supported-version-compatibility`: exercise declared runtime, schema, + adapter, store, and platform version matrix. +7. `representative-latency-throughput`: measure representative latency and + throughput against declared thresholds. +8. `overload-rate-limit-dos`: measure overload, resource exhaustion, rate + limiting, and denial-of-service behavior against declared safety thresholds. +9. `incident-response-exercise`: measure detection, containment, recovery, + communication, and learning exercise. +10. `operational-access-secrets-dependencies-vulnerability-management`: + exercise least privilege, secret lifecycle, dependency inventory, patch + policy, and vulnerability response. + +## Structured report + +Top-level external evidence entry contains: + +- non-generic report `ref`, independent `producer`, HTTPS + `producer_identity`, ISO `observed` date, `relationship` equal to + `independent-third-party`, conflicts list, and exact attestation + `llm-errata-independent-operational-review-v1`; +- result `pass`, `pass-with-findings`, or `fail`; +- full lowercase 40-character `reviewed_commit` and canonical + `surface_digest`; +- `deployment`, `workload`, and `observation_window` declarations; and +- exactly ten `scopes` records. + +Every scope contains its normative token, status `pass` or `fail`, one or more +raw artifact references, one or more measurements, and findings list. Every +measurement contains non-empty metric and unit, finite numeric value and +threshold in same unit, comparator (`<=`, `<`, `>=`, `>`, `==`, or `!=`), and +raw evidence reference. + +Checker evaluates comparators; declared `pass` cannot override failed number. +Structurally complete failed report remains useful evidence but cannot qualify +G6. Generic URL, absent measurement, nine passing scopes, or internally +produced report cannot qualify. + +## Independence and binding + +Producer cannot be author, owner, maintainer, reference implementer, assessed +deployment operator, contributor, repository-controlled agent, or self-review +identity. Conflicts must be declared even when list is empty. + +Surface digest covers this document, security and roadmap documents, checker, +and checker tests using ordered +`relative-path + NUL + raw-bytes + NUL` SHA-256 framing. Reported digest must +match current checkout and named commit. Any drift requires new review. + +Readiness ledger and matrix remain outside digest because they record report +and gate result after review; including them would create self-referential +evidence. Their exact synchronization is checked separately. + +## Evidence workflow + +1. Freeze commit, deployment, workload, thresholds, and observation window. +2. Execute all ten scopes and retain raw artifacts without prohibited content. +3. Independent producer publishes or privately delivers structured report. +4. Maintainer records evidence entry without editing measurements or identity. +5. Run `make readiness`; change G6 only if checker finds qualifying report. + +An invitation, scheduled exercise, CI run, internal agent review, or skeleton +below is not operational evidence. + +```json +{ + "kind": "external", + "result": "fail", + "scopes": [] +} +``` + +Skeleton is deliberately incomplete and non-qualifying. It contains no +invented operator thresholds or measurements. + +## Grounding boundary + +Scope selection draws on incident response, secure development, provenance, +telemetry, and logging guidance pinned in `SOURCES.md`. Those sources do not +certify LLM Errata and do not supply operator-specific numeric thresholds. diff --git a/docs/superpowers/plans/2026-08-12-g6-operational-evidence.md b/docs/superpowers/plans/2026-08-12-g6-operational-evidence.md index 805477c..6b7f10d 100644 --- a/docs/superpowers/plans/2026-08-12-g6-operational-evidence.md +++ b/docs/superpowers/plans/2026-08-12-g6-operational-evidence.md @@ -16,6 +16,8 @@ - Missing thresholds remain `BLOCKED`. - Internal producers, repository CI, invitations, and example reports never qualify. - Exact current checkout and reviewed-commit surface digests must match. +- Readiness ledger and matrix remain outside G6 digest to avoid self-reference; + their status/evidence synchronization is checked separately. - Current verdict remains `NOT_PROD_READY` without genuine external evidence. --- diff --git a/docs/superpowers/specs/2026-08-12-g6-operational-evidence-design.md b/docs/superpowers/specs/2026-08-12-g6-operational-evidence-design.md index 0b88227..1c5537a 100644 --- a/docs/superpowers/specs/2026-08-12-g6-operational-evidence-design.md +++ b/docs/superpowers/specs/2026-08-12-g6-operational-evidence-design.md @@ -125,13 +125,16 @@ valuable evidence but keeps G6 from passing. The digest covers: - `docs/OPERATIONAL_READINESS.md`; -- `PRODUCTION_READINESS.md`; - `SECURITY.md`; - `ROADMAP.md`; -- `readiness/production-readiness.json`; - `scripts/check_readiness.py`; and - `tests/test_readiness.py`. +The readiness ledger and matrix are deliberately outside this digest. They +record the report and gate result after review; including them would make the +evidence self-referential because recording the digest would change the bytes +the digest claims to bind. Their synchronization remains independently checked. + It uses the same ordered `relative path + NUL + raw bytes + NUL` SHA-256 framing as G2. A missing file, unavailable commit, or byte drift blocks qualification. diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index eca0040..20c7e5f 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -75,8 +75,13 @@ "name": "Operational production gates", "class": "external", "status": "BLOCKED", - "criterion": "Security, observability, recovery, compatibility, performance, deployment, and rollback gates pass with current operational evidence.", - "evidence": [{"kind": "repository", "ref": "SECURITY.md"}] + "criterion": "All ten operational scopes pass from one independent report bound to exact commit and deployment, with declared thresholds and measured comparators.", + "evidence": [ + {"kind": "repository", "ref": "docs/OPERATIONAL_READINESS.md"}, + {"kind": "repository", "ref": "SECURITY.md"}, + {"kind": "repository", "ref": "scripts/check_readiness.py"}, + {"kind": "repository", "ref": "tests/test_readiness.py"} + ] } ] } diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index e772f5c..605114e 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -6,9 +6,11 @@ import json import re import hashlib +import math +import operator import subprocess import sys -from datetime import date +from datetime import date, datetime from pathlib import Path from urllib.parse import urlparse @@ -61,6 +63,33 @@ } ) G2_RESULTS = {"pass", "pass-with-findings", "fail"} +G6_ATTESTATION = "llm-errata-independent-operational-review-v1" +G6_SCOPE = frozenset( + { + "deployment-integrity-provenance", + "rollback-exercise", + "backup-recovery-rto-rpo", + "lifecycle-observability-alerting", + "telemetry-privacy-redaction", + "supported-version-compatibility", + "representative-latency-throughput", + "overload-rate-limit-dos", + "incident-response-exercise", + "operational-access-secrets-dependencies-vulnerability-management", + } +) +G6_NON_INDEPENDENT_PRODUCER_RE = re.compile( + r"(?:^|[\s:_-])(author|owner|implementer|operator|contributor|maintainer|thomas|willner|project|reference|local|agent|repository|repo|self)(?:$|[\s:_-])", + re.IGNORECASE, +) +COMPARATORS = { + "<=": operator.le, + "<": operator.lt, + ">=": operator.ge, + ">": operator.gt, + "==": operator.eq, + "!=": operator.ne, +} G2_MATRIX_CURRENT_EVIDENCE = ( "Semantic probes and durable `errata quarantine` checkpoints are internally " "implemented, but Phase 2 remains incomplete: vectors for key rotation, concurrency, invalid targets, " @@ -70,6 +99,14 @@ "Complete listed Phase 2 gaps, then record dated independent external " "conformance-review result covering complete Phase 2 surface." ) +G6_MATRIX_CURRENT_EVIDENCE = ( + "No independent report binds an exact commit and deployment to passing " + "measured comparators for all ten operational scopes." +) +G6_MATRIX_NEXT_EVIDENCE = ( + "One qualifying independent report with declared workload, platform, failure domain, " + "observation window, numeric thresholds, raw artifacts, and passing measurements for every scope." +) class Reporter: @@ -272,6 +309,170 @@ def qualifying_g2_review_evidence( } +def g6_surface_files(root: Path = ROOT) -> tuple[str, ...]: + relative = ( + "docs/OPERATIONAL_READINESS.md", + "SECURITY.md", + "ROADMAP.md", + "scripts/check_readiness.py", + "tests/test_readiness.py", + ) + if any(not (root / path).is_file() for path in relative): + raise OSError("canonical G6 surface is incomplete") + return tuple(sorted(relative)) + + +def g6_surface_digest(root: Path = ROOT) -> str: + return surface_digest_from_bytes( + [(relative, (root / relative).read_bytes()) for relative in g6_surface_files(root)] + ) + + +def g6_surface_digest_at_commit(commit: str, root: Path = ROOT) -> str: + if not reviewed_commit_exists(commit, root): + raise OSError("reviewed commit is unavailable") + entries = [] + for relative in g6_surface_files(root): + result = subprocess.run( + ["git", "show", f"{commit}:{relative}"], cwd=root, + capture_output=True, check=False, + ) + if result.returncode != 0: + raise OSError(f"reviewed commit lacks {relative}") + entries.append((relative, result.stdout)) + return surface_digest_from_bytes(entries) + + +def _exact_dict(value: object, keys: set[str]) -> bool: + return isinstance(value, dict) and set(value) == keys + + +def _nonempty(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _finite_number(value: object) -> bool: + return type(value) in {int, float} and math.isfinite(value) + + +def _iso_timestamp(value: object) -> datetime | None: + if not isinstance(value, str) or not value.endswith("Z"): + return None + try: + return datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError: + return None + + +def _valid_measurement(value: object) -> bool: + keys = {"metric", "value", "unit", "comparator", "threshold", "evidence_ref"} + if not _exact_dict(value, keys): + return False + return ( + _nonempty(value["metric"]) + and _finite_number(value["value"]) + and _nonempty(value["unit"]) + and value["comparator"] in COMPARATORS + and _finite_number(value["threshold"]) + and valid_g2_report_ref(value["evidence_ref"]) + ) + + +def _measurement_passes(value: dict[str, object]) -> bool: + return COMPARATORS[value["comparator"]](value["value"], value["threshold"]) + + +def valid_g6_operational_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: + """Validate complete external G6 report structure and exact bindings.""" + + if not valid_external_evidence(entry, today=today) or not isinstance(entry, dict): + return False + deployment = entry.get("deployment") + workload = entry.get("workload") + window = entry.get("observation_window") + scopes = entry.get("scopes") + reviewed_commit = entry.get("reviewed_commit") + if not ( + entry.get("kind") == "external" + and valid_g2_report_ref(entry.get("ref")) + and G6_NON_INDEPENDENT_PRODUCER_RE.search(entry["producer"]) is None + and valid_g2_identity_ref(entry.get("producer_identity")) + and entry.get("relationship") == "independent-third-party" + and isinstance(entry.get("conflicts"), list) + and entry.get("independence_attestation") == G6_ATTESTATION + and entry.get("result") in G2_RESULTS + and isinstance(reviewed_commit, str) + and re.fullmatch(r"[0-9a-f]{40}", reviewed_commit) is not None + and _exact_dict(deployment, { + "deployment_id", "platform", "environment", "artifact_digest", + "provenance_ref", "deployed_at", + }) + and all(_nonempty(deployment[key]) for key in ("deployment_id", "platform", "environment")) + and isinstance(deployment["artifact_digest"], str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", deployment["artifact_digest"]) is not None + and valid_g2_report_ref(deployment["provenance_ref"]) + and _iso_timestamp(deployment["deployed_at"]) is not None + and _exact_dict(workload, { + "name", "dataset_class", "synthetic_data", "volume", "concurrency", + "duration_seconds", "failure_domain", + }) + and all(_nonempty(workload[key]) for key in ("name", "dataset_class", "failure_domain")) + and isinstance(workload["synthetic_data"], bool) + and all(type(workload[key]) is int and workload[key] > 0 for key in ("volume", "concurrency", "duration_seconds")) + and _exact_dict(window, {"start", "end"}) + and _iso_timestamp(window["start"]) is not None + and _iso_timestamp(window["end"]) is not None + and _iso_timestamp(window["start"]) < _iso_timestamp(window["end"]) + and isinstance(scopes, list) + and len(scopes) == len(G6_SCOPE) + ): + return False + tokens = [] + for scope in scopes: + if not _exact_dict(scope, {"scope", "status", "artifacts", "measurements", "findings"}): + return False + artifacts = scope["artifacts"] + measurements = scope["measurements"] + if not ( + isinstance(scope["scope"], str) + and scope["status"] in {"pass", "fail"} + and isinstance(artifacts, list) and artifacts + and all(valid_g2_report_ref(item) for item in artifacts) + and isinstance(measurements, list) and measurements + and all(_valid_measurement(item) for item in measurements) + and isinstance(scope["findings"], list) + ): + return False + tokens.append(scope["scope"]) + try: + return ( + len(tokens) == len(set(tokens)) + and set(tokens) == G6_SCOPE + and entry.get("surface_digest") == g6_surface_digest(root) + and entry.get("surface_digest") == g6_surface_digest_at_commit(reviewed_commit, root) + ) + except OSError: + return False + + +def qualifying_g6_operational_evidence( + entry: object, *, today: date | None = None, root: Path = ROOT +) -> bool: + if not valid_g6_operational_evidence(entry, today=today, root=root): + return False + return ( + entry["result"] in {"pass", "pass-with-findings"} + and all(scope["status"] == "pass" for scope in entry["scopes"]) + and all( + _measurement_passes(measurement) + for scope in entry["scopes"] + for measurement in scope["measurements"] + ) + ) + + def markdown_row_cells(line: str) -> list[str] | None: if not line.startswith("|") or not line.endswith("|"): return None @@ -410,6 +611,32 @@ def validate_matrix( g2_row is not None and markdown_value(g2_row[4]) == G2_MATRIX_NEXT_EVIDENCE, "G2 matrix requires dated independent external review", ) + g6_gate = next( + (gate for gate in raw_gates if isinstance(gate, dict) and gate.get("id") == "G6"), + None, + ) if isinstance(raw_gates, list) else None + g6_row = next( + (row for row in gate_rows if len(row) == 5 and markdown_value(row[0]) == "G6"), + None, + ) + g6_criterion = g6_gate.get("criterion") if isinstance(g6_gate, dict) else None + reporter.check( + "G6 matrix criterion", + isinstance(g6_criterion, str) + and g6_row is not None + and markdown_value(g6_row[1]) == g6_criterion, + "G6 matrix criterion exactly matches the readiness ledger", + ) + reporter.check( + "G6 matrix current evidence", + g6_row is not None and markdown_value(g6_row[3]) == G6_MATRIX_CURRENT_EVIDENCE, + "G6 matrix states absence of complete measured independent evidence", + ) + reporter.check( + "G6 matrix next evidence", + g6_row is not None and markdown_value(g6_row[4]) == G6_MATRIX_NEXT_EVIDENCE, + "G6 matrix requires all ten measured operational scopes", + ) def validate_ledger( @@ -507,6 +734,7 @@ def validate_ledger( valid_external_entries = 0 valid_g2_reviews = 0 + valid_g6_reports = 0 evidence_valid = isinstance(evidence, list) if isinstance(evidence, list): for index, entry in enumerate(evidence): @@ -554,6 +782,10 @@ def validate_ledger( producer_valid = producer_valid and ( G2_NON_INDEPENDENT_PRODUCER_RE.search(producer) is None ) + if gate_id == "G6" and isinstance(producer, str): + producer_valid = producer_valid and ( + G6_NON_INDEPENDENT_PRODUCER_RE.search(producer) is None + ) reporter.check( f"{entry_name} external reference", reference_valid, @@ -572,11 +804,15 @@ def validate_ledger( entry_valid = valid_external_evidence(entry) if gate_id == "G2": entry_valid = valid_g2_review_evidence(entry, root=ROOT) + elif gate_id == "G6": + entry_valid = valid_g6_operational_evidence(entry, root=ROOT) evidence_valid = evidence_valid and entry_valid if entry_valid: valid_external_entries += 1 if gate_id == "G2" and qualifying_g2_review_evidence(entry, root=ROOT): valid_g2_reviews += 1 + if gate_id == "G6" and qualifying_g6_operational_evidence(entry, root=ROOT): + valid_g6_reports += 1 else: reporter.check(entry_name, False, "kind must be repository or external") evidence_valid = False @@ -584,15 +820,16 @@ def validate_ledger( if not evidence_valid: all_pass = False - external_pass_valid = ( - gate_class != "external" - or status != "PASS" - or (valid_g2_reviews > 0 if gate_id == "G2" else valid_external_entries > 0) + qualifying_external = ( + valid_g2_reviews if gate_id == "G2" + else valid_g6_reports if gate_id == "G6" + else valid_external_entries ) + external_pass_valid = gate_class != "external" or status != "PASS" or qualifying_external > 0 reporter.check( f"{prefix} external PASS evidence", external_pass_valid, - "G2 PASS requires a complete independent Phase 2 review; other external PASS gates require independently observed external evidence", + "external evidence: G2 requires complete conformance review; G6 requires complete measured operational report; other external gates require independent evidence", ) if not external_pass_valid: all_pass = False diff --git a/tests/test_readiness.py b/tests/test_readiness.py index c8d51db..ca93c1c 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import copy import shutil import subprocess import tempfile @@ -10,12 +11,16 @@ from pathlib import Path from scripts.check_readiness import ( + G6_SCOPE, g2_surface_digest, g2_surface_digest_at_commit, g2_surface_files, + g6_surface_digest, + qualifying_g6_operational_evidence, qualifying_g2_review_evidence, valid_external_evidence, valid_g2_review_evidence, + valid_g6_operational_evidence, markdown_value, ) from tests.support import ( @@ -42,6 +47,143 @@ def test_g2_surface_includes_checkpoint_contract_and_tests(self) -> None: self.assertIn("prototype/checkpoints.py", files) self.assertIn("tests/test_checkpoints.py", files) + def test_g6_complete_measured_report_is_commit_and_deployment_bound(self) -> None: + with repo_copy() as source, tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "repository" + shutil.copytree(source, root) + for command in ( + ("git", "init"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Readiness tests"), + ("git", "add", "."), + ("git", "commit", "-m", "operational baseline"), + ): + subprocess.run(command, cwd=root, check=True, capture_output=True) + commit = subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + report = self._complete_g6_report(root, commit) + self.assertTrue(valid_g6_operational_evidence(report, root=root)) + self.assertTrue(qualifying_g6_operational_evidence(report, root=root)) + + def test_g6_failed_comparator_is_valid_but_not_qualifying(self) -> None: + with repo_copy() as source, tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "repository" + shutil.copytree(source, root) + for command in ( + ("git", "init"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Readiness tests"), + ("git", "add", "."), + ("git", "commit", "-m", "operational baseline"), + ): + subprocess.run(command, cwd=root, check=True, capture_output=True) + commit = subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + report = self._complete_g6_report(root, commit) + report["scopes"][0]["measurements"][0]["value"] = 11 + self.assertTrue(valid_g6_operational_evidence(report, root=root)) + self.assertFalse(qualifying_g6_operational_evidence(report, root=root)) + + def test_g6_report_rejects_missing_or_ambiguous_measurement_evidence(self) -> None: + with repo_copy() as source, tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "repository" + shutil.copytree(source, root) + for command in ( + ("git", "init"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Readiness tests"), + ("git", "add", "."), + ("git", "commit", "-m", "operational baseline"), + ): + subprocess.run(command, cwd=root, check=True, capture_output=True) + commit = subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=root, check=True, + capture_output=True, text=True, + ).stdout.strip() + baseline = self._complete_g6_report(root, commit) + + mutations = { + "nine scopes": lambda report: report["scopes"].pop(), + "duplicate scope": lambda report: report["scopes"].__setitem__( + 1, copy.deepcopy(report["scopes"][0]) + ), + "no measurements": lambda report: report["scopes"][0].update(measurements=[]), + "no artifacts": lambda report: report["scopes"][0].update(artifacts=[]), + "missing threshold": lambda report: report["scopes"][0]["measurements"][0].pop("threshold"), + "nonfinite value": lambda report: report["scopes"][0]["measurements"][0].update(value=float("nan")), + "unitless": lambda report: report["scopes"][0]["measurements"][0].update(unit=""), + "invalid comparator": lambda report: report["scopes"][0]["measurements"][0].update(comparator="approximately"), + "missing platform": lambda report: report["deployment"].update(platform=""), + "missing failure domain": lambda report: report["workload"].update(failure_domain=""), + "zero volume": lambda report: report["workload"].update(volume=0), + "reversed window": lambda report: report["observation_window"].update( + start="2026-08-12T11:00:00Z", end="2026-08-12T10:00:00Z" + ), + } + for name, mutate in mutations.items(): + with self.subTest(name=name): + report = copy.deepcopy(baseline) + mutate(report) + self.assertFalse(valid_g6_operational_evidence(report, root=root)) + + @staticmethod + def _complete_g6_report(root: Path, commit: str) -> dict[str, object]: + return { + "kind": "external", + "ref": "https://reviews.example.org/operations/report-1", + "producer": "Independent Reliability Laboratory", + "producer_identity": "https://identity.example.org/reliability-lab", + "observed": "2026-08-12", + "relationship": "independent-third-party", + "conflicts": [], + "independence_attestation": "llm-errata-independent-operational-review-v1", + "result": "pass-with-findings", + "reviewed_commit": commit, + "surface_digest": g6_surface_digest(root), + "deployment": { + "deployment_id": "deploy-20260812-01", + "platform": "linux-amd64", + "environment": "production-like-isolated", + "artifact_digest": "sha256:" + "a" * 64, + "provenance_ref": "https://evidence.example.org/build/1", + "deployed_at": "2026-08-12T08:00:00Z", + }, + "workload": { + "name": "synthetic-correction-mix", + "dataset_class": "synthetic", + "synthetic_data": True, + "volume": 1000, + "concurrency": 10, + "duration_seconds": 600, + "failure_domain": "single-region-store-loss", + }, + "observation_window": { + "start": "2026-08-12T08:00:00Z", + "end": "2026-08-12T10:00:00Z", + }, + "scopes": [ + { + "scope": scope, + "status": "pass", + "artifacts": [f"https://evidence.example.org/{scope}/raw"], + "measurements": [{ + "metric": f"{scope}.gate", + "value": 1, + "unit": "count", + "comparator": "<=", + "threshold": 10, + "evidence_ref": f"https://evidence.example.org/{scope}/measurement", + }], + "findings": [], + } + for scope in sorted(G6_SCOPE) + ], + } + def test_generic_external_evidence_allows_https_root_url(self) -> None: self.assertTrue( valid_external_evidence( @@ -295,6 +437,30 @@ def mutate(root): result = check_after(SCRIPT, mutate) self.assert_rejected_without_traceback(result, "G2 matrix next evidence") + def test_g6_matrix_current_evidence_drift_is_rejected(self) -> None: + def mutate(root): + rewrite( + root / "PRODUCTION_READINESS.md", + "No independent report binds an exact commit and deployment to passing measured comparators for all ten operational scopes.", + "Internal tests prove operations readiness.", + ) + + result = check_after(SCRIPT, mutate) + self.assert_rejected_without_traceback(result, "G6 matrix current evidence") + + def test_generic_external_record_does_not_qualify_as_g6_evidence(self) -> None: + def mutate(payload): + gate = next(gate for gate in payload["gates"] if gate["id"] == "G6") + gate["evidence"].append({ + "kind": "external", + "ref": "https://reviews.example.org/operations/report", + "producer": "Independent Reliability Laboratory", + "observed": "2026-08-12", + }) + + result = self._mutated(mutate) + self.assert_rejected_without_traceback(result, "G6 evidence") + def test_code_formatted_duplicate_matrix_gate_is_rejected(self) -> None: def mutate(root): path = root / "PRODUCTION_READINESS.md" From 50e895fbfec544b16c94caa07bf2d1f4049a42e2 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 22:18:22 +0200 Subject: [PATCH 38/88] feat: complete internal Phase 2 vectors --- CHANGELOG.md | 4 + PRODUCTION_READINESS.md | 2 +- README.md | 2 +- ROADMAP.md | 4 +- prototype/errata.py | 40 ++++++++- prototype/signing.py | 3 +- readiness/production-readiness.json | 2 + scripts/check_readiness.py | 9 +- spec/README.md | 9 ++ spec/erratum.schema.json | 3 +- .../erratum-additional-property-invalid.json | 1 + .../erratum-bad-timestamp-invalid.json | 1 + spec/vectors/erratum-correct-valid.json | 1 + spec/vectors/erratum-erase-valid.json | 1 + ...rratum-erase-with-replacement-invalid.json | 1 + .../erratum-sequence-zero-invalid.json | 1 + spec/vectors/erratum-supersede-valid.json | 1 + ...um-supersede-without-positive-invalid.json | 1 + ...supersede-without-replacement-invalid.json | 1 + .../erratum-unknown-operation-invalid.json | 1 + spec/vectors/manifest.json | 3 +- spec/vectors/protocol-manifest.json | 67 ++++++++++++++ spec/vectors/receipt-binding-mutations.json | 19 ++++ tests/test_errata_feed.py | 21 +++++ tests/test_readiness.py | 12 ++- tests/test_schema.py | 90 ++++++++++++++++++- 26 files changed, 283 insertions(+), 17 deletions(-) create mode 100644 spec/vectors/protocol-manifest.json create mode 100644 spec/vectors/receipt-binding-mutations.json diff --git a/CHANGELOG.md b/CHANGELOG.md index aace09e..54137bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Completed internal Phase 2 conformance surface with owner-key rotation, + rotated-key refusal, concurrent sequence-conflict, invalid-target, + content-free confidentiality, and every-field receipt-binding vectors. G2 + remains `BLOCKED` until dated independent review of exact committed surface. - Added a ten-scope G6 operational evidence contract and fail-closed checker. One independent report must bind exact commit and deployment, declare workload, platform, failure domain, observation window, numeric thresholds, diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 435c2ca..bb6cc87 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Semantic probes and durable `errata quarantine` checkpoints are internally implemented, but Phase 2 remains incomplete: vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded. | Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 is internally complete: schemas, CLI quarantine checkpoints, semantic probes, key rotation, concurrency, invalid-target, confidentiality, and complete receipt-field binding vectors are implemented; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/README.md b/README.md index 3631333..ef256ed 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 is implemented; Phase 2 remains incomplete. Item 6 adds provider-neutral semantic probes with deterministic fixtures and recorded verifier configuration. Explicit `errata quarantine` CLI support now persists a digest-bound checkpoint required by `errata repair`; vectors for key rotation, concurrent events, invalid targets, and confidentiality remain open. Receipt-binding vectors exist for state roots but coverage remains partial. G2 remains `BLOCKED` pending internal Phase 2 completion and a dated independent review of the complete conformance surface. +Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface are implemented. Phase 2 includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, concurrent-conflict and invalid-target cases, content-free confidentiality evidence, and mutation coverage for every signed receipt field. G2 remains `BLOCKED` pending dated independent review of the exact complete conformance surface. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/ROADMAP.md b/ROADMAP.md index 426fafb..d3787c6 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -102,7 +102,7 @@ Kill or redesign the concept if exact lineage cannot survive even the controlled ## Phase 2 — Conformance surface -**Status:** items 1 through 5 are partially implemented and item 6 is internally implemented; Phase 2 is not complete. `spec/` carries schemas, vectors, and deterministic semantic fixtures. `prototype/cli.py` exposes the declared control-plane commands and `semantic-test`; `prototype/checkpoints.py` makes `errata quarantine` durable and requires its state-bound evidence before CLI repair; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. Vectors for key rotation, concurrent events, invalid targets, and confidentiality remain unimplemented. Receipt state-root binding vectors exist, but receipt-binding coverage remains partial. This internal work does not complete G2: a dated independent external conformance review of the complete Phase 2 surface remains required. +**Status:** internally complete, externally unreviewed. `spec/` carries schemas, static vectors, executable stateful vectors, and deterministic semantic fixtures. `prototype/cli.py` exposes the declared control-plane commands and `semantic-test`; `prototype/checkpoints.py` makes `errata quarantine` durable and requires its state-bound evidence before CLI repair; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. `OwnerKeySchedule` binds every event to the key active at its sequence. Stateful vectors execute valid rotation, rotated-key reuse, concurrent sequence conflict, invalid targets, content-free erasure evidence, and mutation of every signed receipt field. This internal work does not complete G2: a dated independent external conformance review of the exact complete Phase 2 surface remains required. Only after the file-backed proof passes: @@ -133,7 +133,7 @@ the erratum, sequence, target, pre-state, adapter inventory, and gated set. `errata repair` re-authenticates and refuses missing, consumed, replayed, or drifted checkpoints before rebuild. -Phase 2 internal implementation is incomplete. Complete the listed vector and receipt-binding gaps before asking an independent reviewer to evaluate the complete conformance surface. G2 remains blocked until that implementation work and a dated independent external review exist. Broader Phase 2 interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). +Phase 2 internal implementation is complete. G2 remains blocked until a dated independent external reviewer evaluates the exact committed surface; local tests and author-directed review cannot satisfy that gate. Broader interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). ## Phase 3 — Interoperability experiment diff --git a/prototype/errata.py b/prototype/errata.py index a8499da..efe6972 100644 --- a/prototype/errata.py +++ b/prototype/errata.py @@ -56,6 +56,32 @@ def __iter__(self) -> Iterator[str]: return iter(sorted(self._roots)) +@dataclass(frozen=True) +class OwnerKeySchedule: + """Owner verification keys activated at explicit feed sequences.""" + + activations: tuple[tuple[int, VerificationKey], ...] + + def __post_init__(self) -> None: + sequences = tuple(item[0] for item in self.activations) + key_ids = tuple(item[1].key_id for item in self.activations) + if ( + not self.activations + or sequences[0] != 1 + or sequences != tuple(sorted(set(sequences))) + or len(key_ids) != len(set(key_ids)) + ): + raise ValueError("key schedule must start at 1 with unique ordered activations") + + def key_for(self, sequence: int) -> VerificationKey: + active = self.activations[0][1] + for activation, key in self.activations: + if activation > sequence: + break + active = key + return active + + @dataclass(frozen=True) class Erratum: erratum_id: str @@ -65,6 +91,7 @@ class Erratum: valid_from: str postconditions: Mapping[str, str] replacement: str | None = None + signing_key_id: str | None = None signature: str | None = None def replace(self, **changes: Any) -> Erratum: @@ -81,6 +108,7 @@ def signable(self) -> dict[str, Any]: "valid_from": self.valid_from, "postconditions": dict(self.postconditions), "replacement": self.replacement, + "signing_key_id": self.signing_key_id, } def to_json(self) -> str: @@ -99,6 +127,7 @@ def from_json(cls, line: str) -> Erratum: valid_from=raw["valid_from"], postconditions=raw["postconditions"], replacement=raw.get("replacement"), + signing_key_id=raw.get("signing_key_id"), signature=raw.get("signature"), ) @@ -143,7 +172,7 @@ def _check_shape(erratum: Erratum, roots: RootRegistry) -> None: def verify_feed( errata: Sequence[Erratum], *, - owner: VerificationKey, + owner: VerificationKey | OwnerKeySchedule, roots: RootRegistry, last_sequence: int = 0, ) -> list[Erratum]: @@ -160,7 +189,14 @@ def verify_feed( previous = last_sequence for erratum in errata: - if erratum.signature is None or not owner.verify( + schedule = owner if isinstance(owner, OwnerKeySchedule) else OwnerKeySchedule(((1, owner),)) + active_key = schedule.key_for(erratum.sequence) + if erratum.signing_key_id != active_key.key_id: + raise FeedError( + f"{erratum.erratum_id}: signing key {erratum.signing_key_id!r} is not " + f"the active key {active_key.key_id!r} at sequence {erratum.sequence}." + ) + if erratum.signature is None or not active_key.verify( erratum.signable(), erratum.signature ): raise FeedError( diff --git a/prototype/signing.py b/prototype/signing.py index 6cadc73..435ce5e 100644 --- a/prototype/signing.py +++ b/prototype/signing.py @@ -116,7 +116,8 @@ def sign(self, payload: dict[str, Any]) -> str: return ed25519.sign(self._seed, canonical_bytes(payload)).hex() def sign_erratum(self, erratum: Erratum) -> Erratum: - return erratum.replace(signature=self.sign(erratum.signable())) + bound = erratum.replace(signing_key_id=self._key_id, signature=None) + return bound.replace(signature=self.sign(bound.signable())) #: Retained so existing fixtures keep working. The name is deliberately no diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index 20c7e5f..36d4b2e 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -37,6 +37,8 @@ {"kind": "repository", "ref": "spec/semantic/probes.json"}, {"kind": "repository", "ref": "spec/semantic/verifier-config.json"}, {"kind": "repository", "ref": "spec/semantic/observations.json"}, + {"kind": "repository", "ref": "spec/vectors/protocol-manifest.json"}, + {"kind": "repository", "ref": "spec/vectors/receipt-binding-mutations.json"}, {"kind": "repository", "ref": "prototype/README.md"}, {"kind": "repository", "ref": "spec/README.md"}, {"kind": "repository", "ref": "REVIEW_REQUEST.md"}, diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 605114e..0d6ea50 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -91,13 +91,12 @@ "!=": operator.ne, } G2_MATRIX_CURRENT_EVIDENCE = ( - "Semantic probes and durable `errata quarantine` checkpoints are internally " - "implemented, but Phase 2 remains incomplete: vectors for key rotation, concurrency, invalid targets, " - "and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded." + "Phase 2 is internally complete: schemas, CLI quarantine checkpoints, semantic probes, " + "key rotation, concurrency, invalid-target, confidentiality, and complete receipt-field " + "binding vectors are implemented; no qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( - "Complete listed Phase 2 gaps, then record dated independent external " - "conformance-review result covering complete Phase 2 surface." + "Dated independent external conformance-review result covering the exact complete Phase 2 surface." ) G6_MATRIX_CURRENT_EVIDENCE = ( "No independent report binds an exact commit and deployment to passing " diff --git a/spec/README.md b/spec/README.md index 0256ec1..d1a5f6f 100644 --- a/spec/README.md +++ b/spec/README.md @@ -12,6 +12,8 @@ than this one can be built and checked against the same contract. | `semantic/probes.json` | Named, strict semantic-probe sets for offline conformance. | | `semantic/verifier-config.json` | Exact synthetic verifier configuration, whose canonical digest binds every observation. | | `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | +| `vectors/protocol-manifest.json` | Executable key-rotation, concurrency, invalid-target, and confidentiality cases that JSON Schema cannot express. | +| `vectors/receipt-binding-mutations.json` | Valid-domain mutations proving every signed receipt field is bound. | ## Offline semantic-probe fixtures @@ -61,6 +63,13 @@ requires each invalid vector to be rejected **for the stated reason**, and requires the invalid set to trip at least four distinct rules, so the suite cannot be one rule wearing many hats. +Stateful vectors are executed, not parsed as schema instances. The feed cases +construct synthetic signed events and key schedules, then require the exact +accept/refuse behavior. Confidentiality case requires forbidden erased content +to remain absent from serialized receipt evidence. Receipt-binding mutations +rebuild a receipt with one field changed while retaining original signature; +verification must fail for every signable field. + The Phase 2 CLI adds a local checkpoint evidence object implemented in `prototype/checkpoints.py`. It is not a third wire schema yet: independent review must settle whether checkpoints are importer-local evidence or a diff --git a/spec/erratum.schema.json b/spec/erratum.schema.json index eb73e03..a1f0801 100644 --- a/spec/erratum.schema.json +++ b/spec/erratum.schema.json @@ -4,7 +4,7 @@ "title": "LLM Errata erratum", "description": "An authorised correction, supersession, or erasure for an exported memory root. This is the signed preimage: the signature is carried alongside and is not part of what it covers.", "type": "object", - "required": ["erratum_id", "sequence", "target_root", "operation", "valid_from", "postconditions"], + "required": ["erratum_id", "sequence", "target_root", "operation", "valid_from", "postconditions", "signing_key_id"], "additionalProperties": false, "properties": { "erratum_id": {"type": "string", "minLength": 1, "pattern": "^[A-Za-z0-9_.:-]+$"}, @@ -23,6 +23,7 @@ "preserve": {"type": "string", "minLength": 1} } }, + "signing_key_id": {"type": "string", "minLength": 1, "description": "Key identifier active for this sequence under the importer's configured owner-key schedule."}, "signature": {"type": ["string", "null"], "pattern": "^[0-9a-f]{128}$"} }, "oneOf": [ diff --git a/spec/vectors/erratum-additional-property-invalid.json b/spec/vectors/erratum-additional-property-invalid.json index 5e4f250..3e2c2b4 100644 --- a/spec/vectors/erratum-additional-property-invalid.json +++ b/spec/vectors/erratum-additional-property-invalid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "supersede", diff --git a/spec/vectors/erratum-bad-timestamp-invalid.json b/spec/vectors/erratum-bad-timestamp-invalid.json index f35d0ed..212f907 100644 --- a/spec/vectors/erratum-bad-timestamp-invalid.json +++ b/spec/vectors/erratum-bad-timestamp-invalid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "1 August 2026", "operation": "supersede", diff --git a/spec/vectors/erratum-correct-valid.json b/spec/vectors/erratum-correct-valid.json index 4468c4c..78dc925 100644 --- a/spec/vectors/erratum-correct-valid.json +++ b/spec/vectors/erratum-correct-valid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "correct", diff --git a/spec/vectors/erratum-erase-valid.json b/spec/vectors/erratum-erase-valid.json index 238c861..b8d0efe 100644 --- a/spec/vectors/erratum-erase-valid.json +++ b/spec/vectors/erratum-erase-valid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "erase", diff --git a/spec/vectors/erratum-erase-with-replacement-invalid.json b/spec/vectors/erratum-erase-with-replacement-invalid.json index 03b556a..7575110 100644 --- a/spec/vectors/erratum-erase-with-replacement-invalid.json +++ b/spec/vectors/erratum-erase-with-replacement-invalid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "erase", diff --git a/spec/vectors/erratum-sequence-zero-invalid.json b/spec/vectors/erratum-sequence-zero-invalid.json index 2f8d98a..3ecdaed 100644 --- a/spec/vectors/erratum-sequence-zero-invalid.json +++ b/spec/vectors/erratum-sequence-zero-invalid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 0, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "supersede", diff --git a/spec/vectors/erratum-supersede-valid.json b/spec/vectors/erratum-supersede-valid.json index 3f17a5b..951bf1a 100644 --- a/spec/vectors/erratum-supersede-valid.json +++ b/spec/vectors/erratum-supersede-valid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "supersede", diff --git a/spec/vectors/erratum-supersede-without-positive-invalid.json b/spec/vectors/erratum-supersede-without-positive-invalid.json index 8ee0bf6..cd3eea4 100644 --- a/spec/vectors/erratum-supersede-without-positive-invalid.json +++ b/spec/vectors/erratum-supersede-without-positive-invalid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "supersede", diff --git a/spec/vectors/erratum-supersede-without-replacement-invalid.json b/spec/vectors/erratum-supersede-without-replacement-invalid.json index 29e4d4a..5497842 100644 --- a/spec/vectors/erratum-supersede-without-replacement-invalid.json +++ b/spec/vectors/erratum-supersede-without-replacement-invalid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "supersede", diff --git a/spec/vectors/erratum-unknown-operation-invalid.json b/spec/vectors/erratum-unknown-operation-invalid.json index 10cc40e..65468c1 100644 --- a/spec/vectors/erratum-unknown-operation-invalid.json +++ b/spec/vectors/erratum-unknown-operation-invalid.json @@ -1,6 +1,7 @@ { "erratum_id": "err_0001", "sequence": 1, + "signing_key_id": "owner", "target_root": "mem_01HX", "valid_from": "2026-08-01T00:00:00Z", "operation": "redact", diff --git a/spec/vectors/manifest.json b/spec/vectors/manifest.json index 79b5951..a045644 100644 --- a/spec/vectors/manifest.json +++ b/spec/vectors/manifest.json @@ -1,5 +1,6 @@ { - "description": "Conformance vectors for the LLM Errata wire schema. Every entry states whether it must validate, and an invalid entry names the text that must appear in the rejection.", + "description": "Conformance vectors for the LLM Errata wire schema. Every entry states whether it must validate, and an invalid entry names the text that must appear in the rejection. Stateful protocol artifacts are declared separately because JSON Schema cannot evaluate ordering, key schedules, target registration, confidentiality, or signature mutation.", + "stateful_vectors": ["protocol-manifest.json", "receipt-binding-mutations.json"], "vectors": [ { "file": "erratum-correct-valid.json", diff --git a/spec/vectors/protocol-manifest.json b/spec/vectors/protocol-manifest.json new file mode 100644 index 0000000..bcb873e --- /dev/null +++ b/spec/vectors/protocol-manifest.json @@ -0,0 +1,67 @@ +{ + "description": "Executable stateful conformance cases. They exercise feed behavior that JSON Schema cannot express.", + "feed_cases": [ + { + "id": "key-rotation-valid", + "why": "each sequence is authenticated by the key active under the declared schedule", + "keys": {"owner-v1": "synthetic-owner-v1", "owner-v2": "synthetic-owner-v2"}, + "schedule": [ + {"sequence": 1, "key_id": "owner-v1"}, + {"sequence": 2, "key_id": "owner-v2"} + ], + "roots": ["mem_01HX"], + "events": [ + {"erratum_id": "err_0001", "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", "signer": "owner-v1"}, + {"erratum_id": "err_0002", "sequence": 2, "target_root": "mem_01HX", "operation": "correct", "signer": "owner-v2"} + ], + "outcome": "accept" + }, + { + "id": "rotated-key-reuse-invalid", + "why": "a rotated-out key cannot authorize later state", + "keys": {"owner-v1": "synthetic-owner-v1", "owner-v2": "synthetic-owner-v2"}, + "schedule": [ + {"sequence": 1, "key_id": "owner-v1"}, + {"sequence": 2, "key_id": "owner-v2"} + ], + "roots": ["mem_01HX"], + "events": [ + {"erratum_id": "err_0001", "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", "signer": "owner-v1"}, + {"erratum_id": "err_0002", "sequence": 2, "target_root": "mem_01HX", "operation": "correct", "signer": "owner-v1"} + ], + "outcome": "reject", + "error_contains": "active key" + }, + { + "id": "concurrent-sequence-conflict-invalid", + "why": "two independently signed events cannot claim the same sequence", + "keys": {"owner-v1": "synthetic-owner-v1"}, + "schedule": [{"sequence": 1, "key_id": "owner-v1"}], + "roots": ["mem_01HX"], + "events": [ + {"erratum_id": "err_concurrent_a", "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", "signer": "owner-v1"}, + {"erratum_id": "err_concurrent_b", "sequence": 1, "target_root": "mem_01HX", "operation": "correct", "signer": "owner-v1"} + ], + "outcome": "reject", + "error_contains": "conflict" + }, + { + "id": "invalid-target-invalid", + "why": "importer cannot guess which local state an unknown root denotes", + "keys": {"owner-v1": "synthetic-owner-v1"}, + "schedule": [{"sequence": 1, "key_id": "owner-v1"}], + "roots": ["mem_01HX"], + "events": [ + {"erratum_id": "err_unknown_target", "sequence": 1, "target_root": "mem_unregistered", "operation": "erase", "signer": "owner-v1"} + ], + "outcome": "reject", + "error_contains": "target" + } + ], + "confidentiality_case": { + "id": "erasure-evidence-content-free", + "target_root": "mem_01HX", + "negative_probe": "vegetarian", + "forbidden_value": "is vegetarian" + } +} diff --git a/spec/vectors/receipt-binding-mutations.json b/spec/vectors/receipt-binding-mutations.json new file mode 100644 index 0000000..de853d8 --- /dev/null +++ b/spec/vectors/receipt-binding-mutations.json @@ -0,0 +1,19 @@ +{ + "description": "One semantically different valid-domain value for every signed receipt field. Reusing the original signature must fail for every mutation.", + "fields": { + "importer": "other-importer", + "erratum_id": "err_other", + "sequence": 2, + "target_root": "mem_other", + "operation": "correct", + "pre_state_root": "00000000000000000000000000000000", + "post_state_root": "11111111111111111111111111111111", + "stores": {"markdown": "unknown"}, + "dispositions": {"markdown": {"fact:diet": "untouched"}}, + "triad": {"negative": "fail", "positive": "pass", "preserve": "pass"}, + "aggregate": "unknown", + "limitations": ["synthetic mutation"], + "history_retained": false, + "adapter_versions": {"markdown": "9.9.9"} + } +} diff --git a/tests/test_errata_feed.py b/tests/test_errata_feed.py index cb02cdc..98e1806 100644 --- a/tests/test_errata_feed.py +++ b/tests/test_errata_feed.py @@ -14,6 +14,7 @@ Erratum, FeedError, Operation, + OwnerKeySchedule, RootRegistry, read_feed, verify_feed, @@ -23,6 +24,8 @@ OWNER = DemoSigner(b"owner-secret") IMPOSTOR = DemoSigner(b"not-the-owner") +OWNER_V1 = DemoSigner(b"owner-v1", key_id="owner-v1") +OWNER_V2 = DemoSigner(b"owner-v2", key_id="owner-v2") ROOTS = RootRegistry({"mem_01HX", "mem_02KP"}) @@ -59,6 +62,24 @@ def test_round_trips_through_jsonl(self) -> None: ["err_0001", "err_0002"], ) + def test_key_rotation_uses_the_key_active_for_each_sequence(self) -> None: + schedule = OwnerKeySchedule(((1, OWNER_V1.public), (2, OWNER_V2.public))) + signed = [ + OWNER_V1.sign_erratum(erratum(1)), + OWNER_V2.sign_erratum(erratum(2)), + ] + accepted = verify_feed(signed, owner=schedule, roots=ROOTS) + self.assertEqual([item.signing_key_id for item in accepted], ["owner-v1", "owner-v2"]) + + def test_rotated_out_key_cannot_sign_a_later_sequence(self) -> None: + schedule = OwnerKeySchedule(((1, OWNER_V1.public), (2, OWNER_V2.public))) + signed = [ + OWNER_V1.sign_erratum(erratum(1)), + OWNER_V1.sign_erratum(erratum(2)), + ] + with self.assertRaisesRegex(FeedError, "active key"): + verify_feed(signed, owner=schedule, roots=ROOTS) + class FeedRejectsForgeryAndReplay(unittest.TestCase): def test_a_forged_signature_is_rejected(self) -> None: diff --git a/tests/test_readiness.py b/tests/test_readiness.py index ca93c1c..8ee4aa8 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -419,18 +419,26 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Semantic probes and durable `errata quarantine` checkpoints are internally implemented, but Phase 2 remains incomplete: vectors for key rotation, concurrency, invalid targets, and confidentiality are absent; receipt-binding vectors are partial; no qualifying independent review is recorded.", + "Phase 2 is internally complete: schemas, CLI quarantine checkpoints, semantic probes, key rotation, concurrency, invalid-target, confidentiality, and complete receipt-field binding vectors are implemented; no qualifying independent review is recorded.", "local tests prove readiness.", ) result = check_after(SCRIPT, mutate) self.assert_rejected_without_traceback(result, "G2 matrix current evidence") + def test_internal_phase2_completion_cannot_upgrade_g2(self) -> None: + def mutate(payload): + gate = next(gate for gate in payload["gates"] if gate["id"] == "G2") + gate["status"] = "PASS" + + result = self._mutated(mutate) + self.assert_rejected_without_traceback(result, "G2 external PASS evidence") + def test_g2_matrix_next_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Complete listed Phase 2 gaps, then record dated independent external conformance-review result covering complete Phase 2 surface.", + "Dated independent external conformance-review result covering the exact complete Phase 2 surface.", "Local tests are enough.", ) diff --git a/tests/test_schema.py b/tests/test_schema.py index 2a7280e..d4befcb 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -18,8 +18,10 @@ from pathlib import Path from prototype import schema +from prototype.adapters import Coverage from prototype.scenario import DIET, build_importer from prototype.signing import DemoSigner +from prototype.errata import Erratum, FeedError, Operation, OwnerKeySchedule, RootRegistry, verify_feed from tests.test_regressions import supersede @@ -127,7 +129,9 @@ def test_every_manifest_entry_has_a_file(self) -> None: ) def test_every_vector_file_is_in_the_manifest(self) -> None: - listed = {entry["file"] for entry in self.manifest["vectors"]} + listed = {entry["file"] for entry in self.manifest["vectors"]} | set( + self.manifest["stateful_vectors"] + ) on_disk = {p.name for p in VECTORS.glob("*.json")} - {"manifest.json"} self.assertEqual(on_disk - listed, set(), "vectors not listed in the manifest") @@ -161,6 +165,90 @@ def test_the_invalid_vectors_are_not_all_rejected_for_one_reason(self) -> None: self.assertGreaterEqual(len(reasons), 4, reasons) +class StatefulProtocolVectors(unittest.TestCase): + def setUp(self) -> None: + self.protocol = json.loads( + (VECTORS / "protocol-manifest.json").read_text(encoding="utf-8") + ) + + def _event(self, record: dict[str, object], signers: dict[str, DemoSigner]) -> Erratum: + operation = Operation(record["operation"]) + postconditions = {"negative": "retired label", "preserve": "retained label"} + replacement = None + if operation is not Operation.ERASE: + replacement = "replacement label" + postconditions["positive"] = "replacement label" + event = Erratum( + erratum_id=record["erratum_id"], + sequence=record["sequence"], + target_root=record["target_root"], + operation=operation, + valid_from="2026-08-12T00:00:00Z", + replacement=replacement, + postconditions=postconditions, + ) + return signers[record["signer"]].sign_erratum(event) + + def test_feed_vectors_execute_named_stateful_rules(self) -> None: + for case in self.protocol["feed_cases"]: + with self.subTest(case=case["id"]): + signers = { + key_id: DemoSigner(seed.encode(), key_id=key_id) + for key_id, seed in case["keys"].items() + } + schedule = OwnerKeySchedule(tuple( + (activation["sequence"], signers[activation["key_id"]].public) + for activation in case["schedule"] + )) + events = [self._event(record, signers) for record in case["events"]] + if case["outcome"] == "accept": + self.assertEqual(len(verify_feed( + events, owner=schedule, roots=RootRegistry(case["roots"]) + )), len(events)) + else: + with self.assertRaisesRegex(FeedError, case["error_contains"]): + verify_feed( + events, owner=schedule, roots=RootRegistry(case["roots"]) + ) + + def test_confidentiality_vector_keeps_forbidden_value_out_of_evidence(self) -> None: + case = self.protocol["confidentiality_case"] + importer = build_importer(OWNER) + receipt = importer.repair(OWNER.sign_erratum(Erratum( + erratum_id="err_confidential_erase", + sequence=1, + target_root=case["target_root"], + operation=Operation.ERASE, + valid_from="2026-08-12T00:00:00Z", + postconditions={"negative": case["negative_probe"], "preserve": "quiet restaurants|moderate budget"}, + ))) + evidence = json.dumps(receipt.to_dict(), sort_keys=True) + self.assertNotIn(case["forbidden_value"], evidence) + + def test_every_receipt_field_mutation_breaks_signature_binding(self) -> None: + fields = json.loads( + (VECTORS / "receipt-binding-mutations.json").read_text(encoding="utf-8") + )["fields"] + receipt = build_importer(OWNER).repair(supersede()) + payload = receipt.to_dict() + for field in fields: + with self.subTest(field=field): + mutated = dict(payload) + mutated[field] = fields[field] + rebuilt = receipt.__class__( + importer=mutated["importer"], erratum_id=mutated["erratum_id"], + sequence=mutated["sequence"], target_root=mutated["target_root"], + operation=mutated["operation"], pre_state_root=mutated["pre_state_root"], + post_state_root=mutated["post_state_root"], + stores={k: Coverage(v) for k, v in mutated["stores"].items()}, + dispositions=mutated["dispositions"], triad=mutated["triad"], + aggregate=Coverage(mutated["aggregate"]), + limitations=mutated["limitations"], history_retained=mutated["history_retained"], + adapter_versions=mutated["adapter_versions"], signature=payload["signature"], + ) + self.assertFalse(rebuilt.verify(OWNER.public)) + + class TheProtoypeAgreesWithItsOwnSchema(unittest.TestCase): def test_a_signed_erratum_validates(self) -> None: payload = supersede().signable() From c83379dbca6adcf125293251f000e43a3f52be3a Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 22:19:17 +0200 Subject: [PATCH 39/88] docs: publish complete Phase 2 review target --- docs/PUBLICATION_LOG.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 44bcdd7..6a50ae9 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -21,14 +21,20 @@ readiness gate. Last verified: 2026-08-12. | Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | | GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | +| Complete Phase 2 review target | Commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2), digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` | Internal Phase 2 surface is complete and locally validated. Published for independent review; publication and internal tests do not satisfy G2. | No targeted invitation has an acceptance or reply recorded. -The conformance review target remains source commit -[`35052868cde8b6f16fc6c20b4136ae0277f9a00e`](https://github.com/thomaswillner/llm-errata/commit/35052868cde8b6f16fc6c20b4136ae0277f9a00e) +The complete conformance review target is source commit +[`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2) with canonical Phase 2 surface digest -`a3133f759620ed942d6cd519d303c0be1554f0193b89e0a141c410aaf4c047b9`. -The implementation and outreach surface at commit +`9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804`. +The digest was recomputed both from the checkout and from `git show` at that +commit with an exact match. Earlier source commit +[`35052868cde8b6f16fc6c20b4136ae0277f9a00e`](https://github.com/thomaswillner/llm-errata/commit/35052868cde8b6f16fc6c20b4136ae0277f9a00e) +and digest `a3133f759620ed942d6cd519d303c0be1554f0193b89e0a141c410aaf4c047b9` +remain historical pre-completion checkpoints, not current review targets. +The earlier implementation and outreach surface at commit [`460883e093b52c99cec697443399da3e03ea2b40`](https://github.com/thomaswillner/llm-errata/commit/460883e093b52c99cec697443399da3e03ea2b40). was the fixed head exercised by GitHub Actions successfully on [Python 3.11](https://github.com/thomaswillner/llm-errata/actions/runs/31619022097/job/94188870893) @@ -70,9 +76,10 @@ interoperability evidence. ## Readiness boundary GitHub calls, invitations, CI results, and internal reviews are not independent -external evidence. Phase 2 remains incomplete: the explicit `errata quarantine` -CLI and vectors for key rotation, concurrent events, invalid targets, and -confidentiality remain absent; receipt state-root binding vectors exist but -receipt-binding coverage is partial. G2 remains `BLOCKED` pending internal Phase -2 completion and a dated independent external review of the complete -conformance surface. Repository verdict remains `NOT_PROD_READY`. +external evidence. Phase 2 is internally complete: explicit quarantine +checkpoints, key rotation, concurrent conflicts, invalid targets, +confidentiality, semantic probes, and every-field receipt binding are covered. +G2 remains `BLOCKED` pending a dated independent external review of exact commit +`50e895fbfec544b16c94caa07bf2d1f4049a42e2` and digest +`9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804`. +Repository verdict remains `NOT_PROD_READY`. From cc4aae11a048c7058e03f8a065b6ddb660a8ab8f Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 22:27:06 +0200 Subject: [PATCH 40/88] docs: strengthen Inspeximus collision record --- CHANGELOG.md | 4 ++++ PRIOR_ART.md | 27 +++++++++++++++++++++++++-- RESEARCH.md | 14 +++++++++++++- SOURCES.md | 3 ++- docs/PUBLICATION_LOG.md | 6 +++++- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54137bb..bbd07d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Corrected the Inspeximus comparison after conflict-disclosed maintainer + feedback and source verification. `retract_lineage` plus `rederive` is a + stronger local quarantine/rebuild collision than the earlier erasure-focused + row; its coverage audit remains weaker than required-store aggregation. - Completed internal Phase 2 conformance surface with owner-key rotation, rotated-key refusal, concurrent sequence-conflict, invalid-target, content-free confidentiality, and every-field receipt-binding vectors. G2 diff --git a/PRIOR_ART.md b/PRIOR_ART.md index 4208e77..849d1cb 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -33,7 +33,7 @@ No individual mechanism is claimed as new. | [EngramSpec](https://engramspec.org/) | **Yes/Partial:** live correction endpoint and pull-based incremental diffs with tombstones; no webhook/push model in v0.1 | No requirement found for quarantine and repair of importer-local summaries, vectors, graphs, or caches | No requirement found | No repair callback found | **Strongest AI-memory transport collision.** It substantially eliminates “portable corrections” as the novel idea, but not importer-local repair conformance. | | [vCon Lifecycle using SCITT](https://datatracker.ietf.org/doc/html/draft-howe-vcon-lifecycle-01) with [SCITT RFC 9943](https://datatracker.ietf.org/doc/rfc9943/) | **Yes:** lifecycle events and acknowledgments across recipients | No AI-memory descendant-repair requirement; workflow implementation is left to applications | No | **Partial:** acknowledgments and SCITT inclusion receipts, not a semantic-repair coverage receipt | **Strongest formal control-plane collision.** It proves that cross-recipient lifecycle events and transparent receipts are not new. | | [Shomei](https://shomei.ai/how-it-works/) ([API](https://shomei.ai/docs/http-api/), [governance](https://shomei.ai/docs/governance-and-receipts/)) | **Partial:** governed correction/update inside the Shomei boundary; no reviewed persistent post-export importer subscription/callback obligation | **Yes locally:** derived lineage, erasure cascades, and explicit external-delete-pending states | **Partial:** bounded governance evidence, but no reviewed mandatory negative + positive + preservation profile for each importer | **Partial:** signed, bounded governance receipts, but not the complete cross-importer callback | **Strongest governed-memory product collision.** It establishes local lineage, lifecycle, honest coverage, and receipts. | -| [Inspeximus](https://github.com/DanceNitra/inspeximus) | **Partial:** local keyed correction/supersession and erasure channel | **Partial:** lineage-aware retraction, echo guard, revert, and residue scanning for explicitly or successfully extractor-keyed assertions | **Partial:** stale-value and preservation-oriented checks; no reviewed mandatory replacement-activation triad across independent stores | **Partial:** signed content-free erasure evidence, not a cross-importer repair callback | **Strongest open-source local correction collision.** Its README also reports that raw conversational prose is rarely keyed reliably, so supersession mostly does not fire there. | +| [Inspeximus](https://github.com/DanceNitra/inspeximus) | **Partial:** local keyed correction/supersession and erasure channel | **Yes locally, within declared lineage:** `retract_lineage` demotes a root and its recorded descendants from default recall, retains them as superseded with `needs_rederivation`, and `rederive` can rebuild against the correction | **Partial:** stale-value and preservation-oriented checks; no reviewed mandatory replacement-activation triad across independent stores | **Partial:** signed content-free erasure evidence plus an `erasure_audit` coverage ratio, but incomplete declared lineage above zero is not itself a failing result and there is no cross-importer callback | **Strongest open-source local correction collision.** It substantially implements quarantine-then-rebuild inside one store. Its documented limit remains recorded lineage and its own boundary, not prior independent importers. | | [MemoRepair](https://arxiv.org/abs/2605.07242v1) | No cross-system delivery | **Yes, as an explicit contract:** descendants withdrawn before repair, republication restricted to validated predecessor-closed successors; invalidated-memory exposure cut from 69.8–94.3% to 0% *given complete influence provenance* | **Partial:** validated republication, no preservation test | No | **Strongest requirement-B collision found.** It independently arrives at quarantine-before-repair. Added 2026-08-07; post-dates the original cutoff. | | [Governed Evolving Memory](https://arxiv.org/abs/2605.26252v1) | No | **Partial:** formal correctness conditions for dependency consistency and provenance preservation | No | No | Argues record-level stores cannot satisfy those conditions. | | [Always-On Agents / AOEP-v0](https://arxiv.org/abs/2606.30306v1) | No | Governance obligations scored, not implemented | **Partial:** a deterministic evaluation contract scoring state mutation and recovery rather than answer quality | No | **Closest conformance-protocol collision.** | @@ -81,7 +81,30 @@ Shomei's [overview](https://shomei.ai/how-it-works/), [HTTP API](https://shomei. That is a substantial collision with any claim that lineage-aware correction, honest coverage, or signed receipts are new. The reviewed material did not establish a persistent obligation for every independently operated prior importer to receive an erratum and return the complete repair-triad callback. -Inspeximus publicly describes keyed supersession, an `echo_guard`, revert, lineage-aware retraction, residue scans, preservation behavior, and content-free signed erasure receipts. These mechanisms apply to explicitly keyed or successfully extractor-keyed assertions. Its README reports that raw conversational prose is rarely keyed reliably and supersession therefore mostly does not fire there. +Inspeximus publicly describes keyed supersession, an `echo_guard`, revert, +lineage-aware retraction, residue scans, preservation behavior, and content-free +signed erasure receipts. At pinned commit +[`4c711f2982911841d86d7ac1989b0ffb866dc891`](https://github.com/DanceNitra/inspeximus/tree/4c711f2982911841d86d7ac1989b0ffb866dc891), +`retract_lineage(subject)` demotes the subject and every descendant reachable +through recorded `derived_from` taint to superseded state, removes them from +default recall, retains them for `include_superseded`, and marks them +`needs_rederivation`; `rederive(subject)` rebuilds eligible descendants against +the corrected root. That is a stronger requirement-B collision than the +previous comparison to `forget_subject` and should be treated as local +quarantine-then-rebuild, not merely deletion. + +The same pinned documentation describes `erasure_audit()` returning +`coverage{records, with_declared_lineage, undeclared_derived, declared_ratio}`. +Its tests force `unaudited` when declared lineage is zero, but a nonzero +incomplete ratio can still return `no_declared_residue`. LLM Errata's required- +store aggregation is therefore stricter: unresolved required coverage cannot +become aggregate success. The Inspeximus maintainer additionally disclosed a +0.0000 declared-lineage ratio in its dogfood store; this repository records that +as interested-party testimony, not independently reproduced code evidence. + +These mechanisms apply to explicitly keyed or successfully extractor-keyed +assertions and recorded lineage. The documented scope is one store, not every +vector index, prompt log, backup, or independently operated importer. LLM Errata does not claim to improve or replace these local systems. It proposes the conformance boundary between an origin and multiple importers. diff --git a/RESEARCH.md b/RESEARCH.md index c21829f..4b591fe 100644 --- a/RESEARCH.md +++ b/RESEARCH.md @@ -130,7 +130,19 @@ The distinction is not that these systems lack cross-recipient events or receipt [Shomei](https://shomei.ai/how-it-works/) documents source-linked recall, policy-checked use, lifecycle events, explicit coverage boundaries, and narrow receipts. The reviewed [HTTP API](https://shomei.ai/docs/http-api/) and [governance documentation](https://shomei.ai/docs/governance-and-receipts/) add correction/update, lineage, erasure-cascade, pending external deletion, and signed-receipt detail. Shomei also states that its receipts do not certify deletion by systems it does not govern. -[Inspeximus](https://github.com/DanceNitra/inspeximus) is a strong open-source local collision: keyed supersession, `echo_guard`, revert, lineage-aware retraction, residue scans, preservation checks, and signed content-free erasure receipts are publicly described. These mechanisms apply to explicitly keyed or successfully extractor-keyed assertions; its README reports that raw conversational prose is rarely keyed reliably and supersession therefore mostly does not fire there. Its documented scope is its own store, not every vector index, prompt log, backup, or independently operated importer. +[Inspeximus](https://github.com/DanceNitra/inspeximus) is a strong open-source +local collision. Its pinned `retract_lineage` and `rederive` lifecycle demotes +recorded descendants from default recall, retains them for re-derivation, and +rebuilds eligible descendants against a corrected root. This is local +quarantine-then-rebuild, not merely erasure. Its `erasure_audit` exposes a +declared-lineage coverage ratio and refuses the zero-lineage case as +`unaudited`, but nonzero incomplete lineage is not itself a failing verdict. +The mechanisms depend on explicitly keyed assertions and recorded lineage, and +the documented scope is its own store, not prior independently operated +importers. The design credits the retract-and-retain lineage to [Doyle's truth- +maintenance work](https://dblp.org/rec/journals/ai/Doyle79.html), reinforcing +that dependency-directed retraction is established prior art rather than an +LLM Errata invention. LLM Errata does not claim these local controls as new. It profiles their missing cross-importer contract. diff --git a/SOURCES.md b/SOURCES.md index 1efa819..40c890c 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -48,7 +48,8 @@ the default branch and will move. | Source | Pin | Verified 2026-08-07 | |---|---|---| -| [Inspeximus](https://github.com/DanceNitra/inspeximus) | `3ba90c47560a53d650b4be47b0146b3a9538dd69` (2026-08-06) | README confirms the keying limitation PRIOR_ART.md relies on: its own benchmark reports 4–8 of 60 non-declarative prose sentences keyed. **This commit post-dates the 2026-08-01 research cutoff** — the repository moved between cutoff and re-verification, which is precisely why this file exists | +| [Inspeximus](https://github.com/DanceNitra/inspeximus) | `4c711f2982911841d86d7ac1989b0ffb866dc891` (observed 2026-08-12) | `docs/API.md` confirms `retract_lineage` demotion, `include_superseded`, `needs_rederivation`, and `rederive`; it also documents `erasure_audit` coverage fields and zero-lineage `unaudited`. Tests confirm zero declared lineage is refused while incomplete nonzero lineage can report `no_declared_residue`. This commit post-dates the 2026-08-01 cutoff and is supplementary collision evidence, not a backdated change to the search corpus. | +| [Doyle, A Truth Maintenance System](https://dblp.org/rec/journals/ai/Doyle79.html) | Artificial Intelligence 12(3), 1979; DBLP `journals/ai/Doyle79` | Stable bibliographic record confirms title and publication. Cited by Inspeximus as prior retract-and-retain lineage. Included to make historical dependency-retraction collision explicit; LLM Errata claims no invention of this mechanism. | | [IAB Data Deletion Request Framework](https://github.com/InteractiveAdvertisingBureau/Data-Subject-Rights/blob/main/Data%20Deletion%20Request%20Framework.md) | `b9418f5394ca91193181a61c567ffbbdff79cdef` (2025-02-06) | Repository unchanged since well before the cutoff | | [Karpathy LLM Wiki idea file](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) | Gist, revision not pinned | Cited only as a presentation precedent, so drift carries no claim risk | diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 6a50ae9..6efd9a8 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -10,6 +10,7 @@ readiness gate. Last verified: 2026-08-12. | Draft implementation and review surface | [PR #8](https://github.com/thomaswillner/llm-errata/pull/8) | Open draft, targeting `agent/prod-readiness`. | | Independent review request | [Issue #4](https://github.com/thomaswillner/llm-errata/issues/4) and its [exact review target](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269378077) | Open call for conformance, novelty, security, and distributed-systems review. | | Targeted independent-review recruitment | [Inspeximus maintainer invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269444341) | Invitation to challenge the stated collision boundary; it is not external evidence. | +| Interested-party prior-art feedback | [Inspeximus maintainer response](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5271988057) | Conflict-disclosed preliminary feedback. Source verification confirmed `retract_lineage` as a stronger local quarantine/rebuild collision and narrowed the coverage comparison. The disclosed dogfood ratio was not independently reproduced. Maintainer offered a future scoped review, but explicitly does not meet G2 independence alone. | | Targeted adapter recruitment | [Remnic invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269523953) | Invitation to become a future independently authored adapter after Phase 2 completion; it is not technical evidence. | | Targeted validator or system recruitment | [ai-memory-mcp invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269524142) | Invitation to choose one separate role: independently produced validator or independently operated system; it is not technical evidence. | | Targeted standards/collision review | [Portable Agent Memory invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269524323) | Invitation to test the conformance/standards collision boundary; it is not review evidence. | @@ -23,7 +24,10 @@ readiness gate. Last verified: 2026-08-12. | GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | | Complete Phase 2 review target | Commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2), digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` | Internal Phase 2 surface is complete and locally validated. Published for independent review; publication and internal tests do not satisfy G2. | -No targeted invitation has an acceptance or reply recorded. +One targeted invitation now has the interested-party response recorded above; +it is not an acceptance of an independent reviewer role and does not satisfy +G2. No adapter, validator, operated-system approval, or CODEOWNER volunteer has +accepted. The complete conformance review target is source commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2) From 29a6b94c5ef42cb2db1915fda97631cd079780db Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 22:35:39 +0200 Subject: [PATCH 41/88] docs: record final Phase 2 publication calls --- docs/PUBLICATION_LOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 6efd9a8..d745266 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -23,6 +23,10 @@ readiness gate. Last verified: 2026-08-12. | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | | GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | | Complete Phase 2 review target | Commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2), digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` | Internal Phase 2 surface is complete and locally validated. Published for independent review; publication and internal tests do not satisfy G2. | +| Final independent-implementation call | [Issue #5 completion update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5272475120) | Publishes the immutable Phase 2 commit and digest for two independent adapters and a separate validator. No implementation has been accepted; G4 remains `BLOCKED`. | +| Final three-system nomination call | [Issue #6 completion update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5272475614) | Publishes the immutable Phase 2 target and exact nomination fields. No system authorization or experiment evidence exists; G5 remains `BLOCKED`. | +| Interested-party review follow-up | [Inspeximus maintainer reply](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5272476094) | Acknowledges the source-grounded correction and points to the final review target. Conflict disclosure remains controlling; a future review from this maintainer cannot satisfy G2 alone. | +| Phase 2 completion announcement | [Discussion #9 completion update](https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648) | Announces internal completion and routes falsifiers to Issues #4, #5, #6, and #10. Publication only; it records no external acceptance or readiness-gate change. | One targeted invitation now has the interested-party response recorded above; it is not an acceptance of an independent reviewer role and does not satisfy @@ -48,6 +52,14 @@ The live [PR #8](https://github.com/thomaswillner/llm-errata/pull/8) identifies later publication-log-only commits and their own CI runs without requiring a self-referential commit identifier in this file. +The later publication head +[`cc4aae11a048c7058e03f8a065b6ddb660a8ab8f`](https://github.com/thomaswillner/llm-errata/commit/cc4aae11a048c7058e03f8a065b6ddb660a8ab8f) +passed [Python 3.11 and Python 3.13](https://github.com/thomaswillner/llm-errata/actions/runs/31637838807) +CI. Python 3.11 initially failed after all 250 test assertions ran, during +`TemporaryDirectory` removal of a Git object directory; one bounded rerun of +the failed job passed. This classifies the first result as an observed cleanup +race rather than suppressing it or treating it as protocol evidence. + ## Direct email outreach | Recipient | Subject | Status | From d4c0141d32ebf8c9bdd0b9a4b7cdf8ff4730d92b Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Wed, 12 Aug 2026 22:38:52 +0200 Subject: [PATCH 42/88] docs: record six-step readiness checkpoint --- docs/PUBLICATION_LOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index d745266..6b03463 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -27,6 +27,7 @@ readiness gate. Last verified: 2026-08-12. | Final three-system nomination call | [Issue #6 completion update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5272475614) | Publishes the immutable Phase 2 target and exact nomination fields. No system authorization or experiment evidence exists; G5 remains `BLOCKED`. | | Interested-party review follow-up | [Inspeximus maintainer reply](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5272476094) | Acknowledges the source-grounded correction and points to the final review target. Conflict disclosure remains controlling; a future review from this maintainer cannot satisfy G2 alone. | | Phase 2 completion announcement | [Discussion #9 completion update](https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648) | Announces internal completion and routes falsifiers to Issues #4, #5, #6, and #10. Publication only; it records no external acceptance or readiness-gate change. | +| Six-step readiness checkpoint | [PR #8 checkpoint](https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272526975) | Audits every approved production-readiness step against exact internal and external evidence. Records `NOT_PROD_READY`, the external blockers, PR #3 status, validation, and publication boundaries without claiming certification. | One targeted invitation now has the interested-party response recorded above; it is not an acceptance of an independent reviewer role and does not satisfy From 6c24d5b45a4ab677d4bde03c9138dee23aad4bf6 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 00:06:35 +0200 Subject: [PATCH 43/88] docs: approve specification attribution design --- ...ation-implementation-attribution-design.md | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-12-specification-implementation-attribution-design.md diff --git a/docs/superpowers/specs/2026-08-12-specification-implementation-attribution-design.md b/docs/superpowers/specs/2026-08-12-specification-implementation-attribution-design.md new file mode 100644 index 0000000..15f3072 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-specification-implementation-attribution-design.md @@ -0,0 +1,167 @@ +# Specification Implementation and Attribution Design + +**Date:** 2026-08-12 + +**Status:** Approved design; licence implementation requires owner review of +this written specification before execution. + +## Decision + +Adopt a dual-boundary licence for version 0.3.0 and later: + +1. grant an irrevocable, worldwide, royalty-free, non-exclusive right to + implement the LLM Errata specification in commercial and non-commercial + products and services; and +2. retain the current personal-use restrictions for the repository's reference + implementation and other source code unless a separate licence applies. + +Every product or service that implements the specification must attribute +**LLM Errata** and **Thomas Willner**, regardless of whether the use is +commercial or non-commercial. + +This design does not change the irrevocable Apache-2.0 rights already granted +for version 0.2.0 and earlier. + +## Licensed boundary + +The implementation grant covers the normative specification and conformance +materials needed to build an interoperable implementation: + +- `spec/`, excluding third-party material under `spec/vendor/`; +- the normative protocol requirements incorporated by reference from + `README.md`, `IDEA.md`, `ROADMAP.md`, `SECURITY.md`, and `THREAT_MODEL.md`; +- published schemas, canonical examples, and conformance vectors; and +- the right to create, use, modify, distribute, host, and commercialize an + independent implementation of those requirements. + +The grant does not permit copying or distributing repository implementation +code unless another licence or written permission allows it. In particular, +`prototype/`, `scripts/`, and `tests/` remain under the personal-use source-code +licence. Implementers may run published conformance tests for evaluation where +the licence expressly permits it, but must independently author product code. + +Third-party files remain governed only by their own licences. Nothing in this +design relicenses them. + +## Attribution requirement + +Every commercial or non-commercial product or service implementing any +material part of the specification must provide this credit, or a +substantially equivalent credit preserving all three elements: + +> Implements the LLM Errata specification by Thomas Willner — +> https://github.com/thomaswillner/llm-errata + +The attribution must be reasonably accessible to users or recipients in at +least one ordinary product location, such as: + +- an About or Legal screen; +- product or API documentation; +- acknowledgements or third-party notices; or +- a distributed `NOTICE` file. + +A source-code comment that is not ordinarily accessible to product users or +recipients does not satisfy the requirement. The licence does not require a +logo, home-screen badge, advertising statement, or fixed visual placement. + +The attribution must not imply that Thomas Willner endorses, sponsors, +certifies, audits, or accepts responsibility for an implementation. Conformance +claims remain subject to the repository's evidence requirements. + +## Grant properties and retained rights + +The specification implementation grant is: + +- irrevocable for specification versions released under it; +- worldwide, royalty-free, and non-exclusive; +- sublicensable only as needed to distribute, license, sell, or operate an + implementation, provided downstream recipients remain subject to the + attribution, notice-preservation, and no-false-endorsement conditions; +- available for commercial, governmental, academic, standards, research, and + personal use; and +- conditioned on attribution, preservation of notices, no false endorsement, + and compliance with third-party rights. + +The grant provides no trademark rights beyond nominative use needed for the +required attribution. It provides no patent licence unless the final licence +text explicitly says otherwise. No party receives certification status merely +by implementing the specification. + +The copyright holder retains all rights in reference source code not expressly +granted here and may offer separate commercial, source-code, certification, or +support terms. + +## Repository changes + +Implementation will update these surfaces together: + +- `LICENSE`: add the irrevocable specification implementation grant, exact + attribution condition, path boundary, no-endorsement rule, retained + source-code restrictions, warranty disclaimer, and historical Apache grant; +- `README.md`: state the dual boundary without calling the source code open + source; +- `NOTICE`: provide canonical attribution wording and historical licence note; +- `IMPLEMENTATION_CALL.md` and `INDEPENDENT_IMPLEMENTATION.md`: remove the + per-implementer permission gate for clean-room specification implementations + and require the canonical attribution; +- `CONTRIBUTING.md` and `SECURITY.md` where licence or support wording is + affected; +- `SOURCES.md` only if new external legal or licensing references are added; +- repository validators and focused negative tests so licence drift fails; and +- the publication log and GitHub maintainer reply after the final commit and + validation target exist. + +## Protocol feedback incorporated in the same change set + +The maintainer feedback also identified two documentation defects independent +of licensing: + +1. normative requirement 4 combines two separate properties; documentation + will distinguish signature authenticity from coverage truthfulness while + preserving the four-part conjunction; and +2. cryptography qualification understates refusal-path evidence; it will lead + with malformed-length, non-canonical-scalar, tampered-message, + tampered-signature, and wrong-key refusal before compatibility vectors. + +These clarifications change the canonical Phase 2 surface. Implementation must +compute and publish a new frozen G2 commit and digest. The previous target +remains a historical checkpoint, not the current review target. + +## Validation and publication + +Before publication: + +1. add focused tests that fail on missing implementation rights, missing + commercial/non-commercial attribution, a source-code relicensing overreach, + or language implying endorsement; +2. run `make check` and `make cli-demo`; +3. run `make links` only if citations change, reporting network failures + separately from deterministic checks; +4. recompute the canonical G2 surface digest from the checkout and committed + target; +5. commit and push only the reviewed repository delta; +6. require exact-head Python 3.11 and 3.13 CI; and +7. reply to the maintainer in natural language with accepted recommendations, + retained boundaries, exact commit and digest, and the continuing conflict + disclosure. + +The repository remains `NOT_PROD_READY`. A licence grant, attribution, public +reply, or interested-party review does not satisfy G2 or any other external +readiness gate. + +## Risks and mitigations + +- **Legal ambiguity:** this is an engineering licence design, not legal advice. + Use plain definitions, explicit path boundaries, and professional legal + review before relying on it for material commercial enforcement. +- **Accidental source-code relicensing:** tests and documentation must keep + independent specification implementation separate from copying `prototype/`. +- **Downstream attribution loss:** product distribution and service-operation + rights must carry the attribution and no-endorsement conditions to recipients + and operators without extending to unrelated reference code. +- **Attribution burden:** allow several ordinary product locations; do not + require prominent UI branding. +- **False endorsement:** require attribution but explicitly prohibit language + suggesting certification or approval. +- **Review-target drift:** publish one new exact commit and G2 digest after all + normative changes are committed. From e18ba88ffcf48ad0e296ff72e1ea29589303d59e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 00:09:19 +0200 Subject: [PATCH 44/88] docs: plan attributed specification grant --- ...ication-attribution-and-review-feedback.md | 414 ++++++++++++++++++ 1 file changed, 414 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-13-specification-attribution-and-review-feedback.md diff --git a/docs/superpowers/plans/2026-08-13-specification-attribution-and-review-feedback.md b/docs/superpowers/plans/2026-08-13-specification-attribution-and-review-feedback.md new file mode 100644 index 0000000..75999d8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-specification-attribution-and-review-feedback.md @@ -0,0 +1,414 @@ +# Specification Attribution and Review Feedback Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Grant irrevocable commercial and non-commercial implementation rights for the LLM Errata specification with mandatory Thomas Willner attribution, preserve personal-use restrictions on reference code, and incorporate the Inspeximus maintainer's remaining protocol and cryptography recommendations. + +**Architecture:** Keep one custom `LICENSE` with two explicit scopes: a specification implementation grant and a personal-use source-code grant. Enforce the boundary with dependency-free repository checks and negative tests. Clarify, without renumbering the four-part conjunction, that requirement D contains independently evaluated authenticity and coverage-truthfulness properties; then freeze and publish a new G2 review commit and digest. + +**Tech Stack:** Markdown, Python 3.11/3.13 standard library, `unittest`, existing repository validators, Git, GitHub CLI. + +## Global Constraints + +- The specification implementation grant is irrevocable, worldwide, royalty-free, and non-exclusive for versions released under it. +- Commercial, governmental, academic, standards, research, personal, hosted-service, and distributed-product implementations are permitted. +- Every commercial or non-commercial product or service implementing a material part of the specification must name `LLM Errata`, `Thomas Willner`, and `https://github.com/thomaswillner/llm-errata` in an ordinarily accessible About, Legal, documentation, acknowledgements, or NOTICE location. +- Attribution must not imply endorsement, sponsorship, certification, audit, or responsibility. +- `prototype/`, `scripts/`, and `tests/` remain personal-use source code; independent product code may implement `spec/` but may not copy reference code without separate permission. +- `spec/vendor/` remains governed only by third-party licences. +- Version 0.2.0 and earlier retain irrevocable Apache-2.0 rights. +- No patent licence or certification status is granted. +- Preserve correction, supersession, erasure, quarantine-before-repair, repair triad, content-free erasure evidence, and fail-closed coverage semantics. +- Repository verdict remains `NOT_PROD_READY`; interested-party feedback cannot satisfy G2 independently. + +--- + +### Task 1: Machine-Enforced Dual Licence Contract + +**Files:** +- Modify: `tests/test_validate_repo.py` +- Modify: `scripts/validate_repo.py` +- Modify: `LICENSE` + +**Interfaces:** +- Consumes: `read_utf8()`, `Reporter.check()`, and `check_publication_metadata()` in `scripts/validate_repo.py`. +- Produces: deterministic `license and notice` validation covering implementation rights, attribution, source-code boundary, no endorsement, no patent grant, and historical Apache rights. + +- [ ] **Step 1: Write failing negative tests** + +Add tests that mutate one exact licence property at a time: + +```python +def test_specification_implementation_grant_cannot_be_removed(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "LICENSE", + "commercial and non-commercial products and services", + "personal non-commercial experiments", + ) + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("license and notice", result.stdout) + +def test_required_product_attribution_cannot_be_removed(self) -> None: + def mutate(root: Path) -> None: + rewrite(root / "LICENSE", "Thomas Willner", "the author") + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("license and notice", result.stdout) + +def test_reference_code_cannot_be_relicensed_by_the_specification_grant(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "LICENSE", + "does not cover `prototype/`, `scripts/`, or `tests/`", + "also covers `prototype/`, `scripts/`, and `tests/`", + ) + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("license and notice", result.stdout) + +def test_false_endorsement_protection_cannot_be_removed(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "LICENSE", + "does not imply endorsement, sponsorship, certification, or audit", + "implies certification by the author", + ) + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("license and notice", result.stdout) +``` + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: + +```bash +python3 -m unittest tests.test_validate_repo -v +``` + +Expected: the new tests fail because current licence lacks the specification grant and canonical attribution contract. + +- [ ] **Step 3: Replace `LICENSE` with explicit dual-scope terms** + +Write these sections in plain language: + +1. historical Apache-2.0 rights; +2. definitions for `Specification Materials`, `Reference Code`, `Implementation`, and `Product or Service`; +3. irrevocable specification implementation grant including necessary downstream sublicensing; +4. exact product attribution and accessible-location rule; +5. no false endorsement, trademark, patent, or certification grant; +6. personal-use reference-code grant and restrictions; +7. third-party materials, contributions, termination for condition breach, and warranty disclaimer. + +Canonical credit: + +```text +Implements the LLM Errata specification by Thomas Willner — +https://github.com/thomaswillner/llm-errata +``` + +- [ ] **Step 4: Strengthen validator** + +Replace the old `Personal Use Licence`-only predicate with exact required clauses: + +```python +required_license = ( + "LLM Errata Specification Implementation and Personal Use Licence", + "commercial and non-commercial products and services", + "Implements the LLM Errata specification by Thomas Willner", + "https://github.com/thomaswillner/llm-errata", + "does not cover `prototype/`, `scripts/`, or `tests/`", + "does not imply endorsement, sponsorship, certification, or audit", + "No patent rights are granted", + "Apache License 2.0", + "irrevocable", +) +``` + +Require every string plus matching Thomas Willner copyright in `LICENSE` and `NOTICE`. + +- [ ] **Step 5: Run focused tests and verify GREEN** + +Run: + +```bash +python3 -m unittest tests.test_validate_repo -v +``` + +Expected: all validator tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add LICENSE scripts/validate_repo.py tests/test_validate_repo.py +git commit -m "license: permit attributed specification implementations" +``` + +### Task 2: Public Licence and Adoption Documents + +**Files:** +- Modify: `README.md` +- Modify: `NOTICE` +- Modify: `INDEPENDENT_IMPLEMENTATION.md` +- Modify: `CONTRIBUTING.md` +- Modify: `SECURITY.md` +- Modify: `CHANGELOG.md` +- Test: `tests/test_validate_repo.py` + +**Interfaces:** +- Consumes: dual-scope terms from Task 1. +- Produces: synchronized user-facing adoption instructions and canonical attribution. + +- [ ] **Step 1: Add failing synchronization tests** + +Add one table-driven test asserting required phrases in each surface: + +```python +def test_public_documents_state_the_dual_license_boundary(self) -> None: + with repo_copy() as root: + required = { + "README.md": ("commercial and non-commercial", "Thomas Willner"), + "NOTICE": ("LLM Errata specification by Thomas Willner",), + "INDEPENDENT_IMPLEMENTATION.md": ("No per-implementer permission",), + "CONTRIBUTING.md": ("independently authored implementation",), + } + for name, phrases in required.items(): + text = (root / name).read_text(encoding="utf-8") + for phrase in phrases: + self.assertIn(phrase, text, name) +``` + +- [ ] **Step 2: Run focused test and verify RED** + +Run the exact new test. Expected: FAIL on current personal-use-only wording. + +- [ ] **Step 3: Synchronize documents** + +- `README.md`: describe repository as source-available; permit attributed independent specification implementations; retain personal-use reference-code boundary. +- `NOTICE`: add canonical credit and dual-scope notice. +- `INDEPENDENT_IMPLEMENTATION.md`: remove written-permission intake; state that clean-room implementations need no individual grant and must preserve attribution. +- `CONTRIBUTING.md`: distinguish implementation reports from copying reference code. +- `SECURITY.md`: state security support does not certify third-party implementations and licence attribution does not imply endorsement. +- `CHANGELOG.md`: record accepted licensing/adoption feedback and retained limits. + +- [ ] **Step 4: Run focused tests and validator** + +```bash +python3 -m unittest tests.test_validate_repo -v +python3 scripts/validate_repo.py +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add README.md NOTICE INDEPENDENT_IMPLEMENTATION.md CONTRIBUTING.md SECURITY.md CHANGELOG.md tests/test_validate_repo.py +git commit -m "docs: publish attributed implementation grant" +``` + +### Task 3: Separate Receipt Authenticity from Coverage Truthfulness + +**Files:** +- Modify: `scripts/claim_guard.py` +- Modify: `tests/test_claim_guard.py` +- Modify: `README.md` +- Modify: `IDEA.md` +- Modify: `RESEARCH.md` +- Modify: `PRIOR_ART.md` +- Modify: `REVIEW_REQUEST.md` +- Modify: `ROADMAP.md` +- Modify: `spec/README.md` +- Modify: `CHANGELOG.md` + +**Interfaces:** +- Consumes: existing receipt schema and `Receipt.verify()`/`aggregate_coverage()` behavior. +- Produces: one four-part conjunction whose callback requirement has two independently testable subproperties. + +- [ ] **Step 1: Add failing claim-guard tests** + +Add exact anchors and negative tests for: + +```text +Signature validity authenticates the importer and receipt bytes; it does not establish that the reported coverage is truthful. +Coverage truthfulness requires the signed stores, aggregate, and limitations to match the declared required scope without upgrading missing or opaque evidence. +``` + +Each test replaces one sentence with its unsafe inversion and expects `claim_guard.py` exit `1`. + +- [ ] **Step 2: Run focused claim tests and verify RED** + +```bash +python3 -m unittest tests.test_claim_guard -v +``` + +Expected: new anchor tests fail because the canonical sentences are absent. + +- [ ] **Step 3: Update normative prose consistently** + +Keep four top-level requirements. Rename D to `Authenticated and coverage-truthful callback`, then define: + +- `D1 Authenticity`: valid signature over event, importer, pre/post state, stores, aggregate, and limitations; +- `D2 Coverage truthfulness`: signed coverage matches declared required scope; missing, opaque, skipped, or failed evidence cannot become success. + +State the two canonical sentences above in `IDEA.md`, add matching concise language elsewhere, and preserve existing bounded novelty wording. + +- [ ] **Step 4: Add exact anchors to `claim_guard.py`** + +Guard both canonical sentences in `IDEA.md`. Do not weaken existing anchors or add quotation exceptions. + +- [ ] **Step 5: Run claim and schema/controller suites** + +```bash +python3 -m unittest tests.test_claim_guard tests.test_schema tests.test_controller -v +``` + +Expected: PASS; runtime/schema behavior remains unchanged. + +- [ ] **Step 6: Commit** + +```bash +git add scripts/claim_guard.py tests/test_claim_guard.py README.md IDEA.md RESEARCH.md PRIOR_ART.md REVIEW_REQUEST.md ROADMAP.md spec/README.md CHANGELOG.md +git commit -m "docs: separate receipt authenticity from coverage truth" +``` + +### Task 4: Promote Cryptographic Refusal Evidence + +**Files:** +- Modify: `docs/CRYPTOGRAPHY_QUALIFICATION.md` +- Modify: `tests/test_readiness.py` +- Modify: `CHANGELOG.md` + +**Interfaces:** +- Consumes: `VerificationRefusesBadInput` in `tests/test_ed25519.py`. +- Produces: qualification record that distinguishes refusal-path evidence from signing compatibility and production assurance. + +- [ ] **Step 1: Add failing readiness-document test** + +Require the qualification to enumerate: + +```python +for phrase in ( + "non-canonical scalar", + "malformed public-key and signature lengths", + "tampered message", + "tampered signature", + "wrong public key", +): + self.assertIn(phrase, qualification) +``` + +- [ ] **Step 2: Run focused test and verify RED** + +Run the exact test. Expected: FAIL for absent refusal-evidence summary. + +- [ ] **Step 3: Rewrite qualification evidence ordering** + +Add `### Verification refusal evidence` before PyCA compatibility. Explain that refusal tests are stronger security-relevant reference evidence than successful vector reproduction, while neither proves constant-time behavior or production readiness. Link each phrase to the existing test method name without adding external citations. + +- [ ] **Step 4: Run focused readiness and Ed25519 tests** + +```bash +python3 -m unittest tests.test_readiness tests.test_ed25519 -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add docs/CRYPTOGRAPHY_QUALIFICATION.md tests/test_readiness.py CHANGELOG.md +git commit -m "docs: foreground cryptographic refusal evidence" +``` + +### Task 5: Freeze, Publish, and Verify the New Review Target + +**Files:** +- Modify: `docs/PUBLICATION_LOG.md` +- Modify: ignored `.scratch/g2-publication/loop-state.yaml` +- Create: ignored `.scratch/g2-publication/maintainer-reply.md` + +**Interfaces:** +- Consumes: committed normative surface from Tasks 1-4 and `g2_surface_digest_at_commit()`. +- Produces: new immutable G2 commit/digest, natural maintainer reply, publication log entries, pushed branch, and exact-head CI evidence. + +- [ ] **Step 1: Run complete deterministic validation** + +```bash +make check +make cli-demo +git diff --check +``` + +Expected: 250 or more tests pass; demo exits through its expected wrapper result; no whitespace errors. + +- [ ] **Step 2: Freeze and verify source target** + +Record current `git rev-parse HEAD` as the new review commit. Compute: + +```bash +python3 -c 'from scripts.check_readiness import g2_surface_digest; print(g2_surface_digest())' +review_commit=$(git rev-parse HEAD) +python3 -c 'import sys; from scripts.check_readiness import g2_surface_digest_at_commit; print(g2_surface_digest_at_commit(sys.argv[1]))' "$review_commit" +``` + +Expected: identical 64-character digests. + +- [ ] **Step 3: Draft a human maintainer reply** + +Reply substance: + +- thank them for checking code and correcting their own earlier assumption; +- say `retract_lineage`/`rederive` and Doyle were incorporated; +- say requirement D now separates authenticity from coverage truthfulness; +- say the cryptography record now foregrounds refusal paths; +- answer licence question directly: revocability was deliberate for reference code, but their adoption concern was persuasive, so independent specification implementations now receive an irrevocable royalty-free grant with mandatory Thomas Willner/LLM Errata attribution; +- preserve no-endorsement and conflict boundaries; +- provide exact commit, digest, and review contract; +- invite their proposed quarantine/equivocation/undeclared-lineage review. + +Avoid corporate boilerplate, exaggerated praise, or readiness claims. + +- [ ] **Step 4: Push source target and verify CI** + +```bash +git push origin agent/g2-publication +review_commit=$(git rev-parse HEAD) +run_id=$(gh run list --repo thomaswillner/llm-errata --branch agent/g2-publication --limit 10 --json databaseId,headSha --jq '.[] | select(.headSha == "'"$review_commit"'") | .databaseId' | head -1) +test -n "$run_id" +gh run watch "$run_id" --repo thomaswillner/llm-errata --exit-status +``` + +Expected: Python 3.11 and 3.13 pass for exact source commit. + +- [ ] **Step 5: Publish reply and record URL** + +```bash +gh issue comment 4 --repo thomaswillner/llm-errata --body-file .scratch/g2-publication/maintainer-reply.md +``` + +Add reply URL, new G2 target, digest, accepted recommendations, and evidence boundary to `docs/PUBLICATION_LOG.md`. + +- [ ] **Step 6: Commit and push publication record** + +```bash +git add docs/PUBLICATION_LOG.md +git commit -m "docs: publish attributed review target" +git push origin agent/g2-publication +``` + +- [ ] **Step 7: Final verification** + +Require: + +- clean worktree; +- local HEAD equals `origin/agent/g2-publication`; +- exact-head Python 3.11 and 3.13 CI green; +- PR #8 remains open and points to exact head; +- G2 remains `BLOCKED` with no qualifying independent review. + +- [ ] **Step 8: Commit no further self-referential publication changes** + +Stop after reporting final commit, frozen source commit/digest, public reply URL, validations, and remaining external blockers. From 0a4aed1bccbd59d63d96a65c220a1b54b74104e8 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 00:12:51 +0200 Subject: [PATCH 45/88] license: permit attributed specification implementations --- LICENSE | 113 ++++++++++++++++++++++++------------ scripts/validate_repo.py | 25 +++++--- tests/test_validate_repo.py | 35 +++++++++++ 3 files changed, 128 insertions(+), 45 deletions(-) diff --git a/LICENSE b/LICENSE index fb52887..d1e21e0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -LLM Errata — Personal Use Licence +LLM Errata Specification Implementation and Personal Use Licence Copyright 2026 Thomas Rainer Willner. All rights reserved. This licence applies to version 0.3.0 and later. Version 0.2.0 and earlier were @@ -6,59 +6,98 @@ published under the Apache License 2.0. That grant is irrevocable for those releases: anyone who obtained them retains their Apache-2.0 rights to those versions permanently. This licence does not, and cannot, withdraw it. -1. GRANT +1. DEFINITIONS - Subject to the conditions below, you are granted a personal, non-exclusive, - non-transferable, revocable licence to: + "Specification Materials" means the normative requirements incorporated by + reference from README.md, IDEA.md, ROADMAP.md, SECURITY.md, and + THREAT_MODEL.md, plus files under `spec/` other than `spec/vendor/`. + + "Reference Code" means files under `prototype/`, `scripts/`, and `tests/`, + together with other repository software not expressly included in the + Specification Materials. + + "Implementation" means independently authored software or services that + implement a material part of the Specification Materials without copying + Reference Code. + +2. SPECIFICATION IMPLEMENTATION GRANT - a) read, study, and evaluate this work; - b) run the software on your own machine for personal, non-commercial - evaluation, study, or research; - c) quote and cite this work with attribution, including in academic - publications, standards discussions, and public commentary. + Subject to sections 3 through 7, the copyright holder grants you an + irrevocable, worldwide, royalty-free, non-exclusive licence to create, use, + modify, distribute, host, and commercialize independent Implementations in + commercial and non-commercial products and services. -2. CONDITIONS AND RESTRICTIONS + You may sublicense this right only as needed to distribute, license, sell, + or operate an Implementation. You must require downstream recipients and + operators to preserve the attribution and no-false-endorsement conditions + in section 3. - Without prior written permission from the copyright holder, you may not: + This implementation grant does not cover `prototype/`, `scripts/`, or `tests/`. + It does not grant permission to copy, modify, distribute, + sublicense, sell, or host Reference Code. Those acts require separate + written permission unless another licence expressly applies. - a) use this work, in whole or in part, for any commercial purpose, including - internal use within a commercial or governmental organisation; - b) redistribute, publish, sublicense, sell, or host this work or any - substantial portion of it; - c) create or distribute derivative works, adaptations, or translations; - d) implement the specifications in `spec/` in any product or service, - whether or not that product is distributed; - e) remove or alter this licence, the copyright notice, or the attribution - of authorship. +3. REQUIRED ATTRIBUTION AND NO FALSE ENDORSEMENT - Permission for any of the above may be requested from the author. It is not - unreasonably withheld for research and standards work. + Every commercial or non-commercial product or service implementing a + material part of the Specification Materials must provide this credit, or a + substantially equivalent credit preserving the project name, author name, + and repository URL: -3. THIRD-PARTY MATERIAL + Implements the LLM Errata specification by Thomas Willner — + https://github.com/thomaswillner/llm-errata - Files under `spec/vendor/` are third-party works redistributed under their - own licences and are NOT covered by this licence. See the licence file in - each vendored directory. The JSON-Schema-Test-Suite is Copyright (c) 2012 - Julian Berman and is licensed under the MIT Licence. + The credit must be reasonably accessible to users or recipients in at least + one ordinary product location, such as an About or Legal screen, product or + API documentation, acknowledgements, third-party notices, or a distributed + NOTICE file. A source-code comment unavailable to ordinary users or + recipients is insufficient. No logo, home-screen badge, advertising + statement, or fixed visual placement is required. + + Attribution does not imply endorsement, sponsorship, certification, or audit + by Thomas Willner. You must not state or imply otherwise. + +4. PERSONAL USE OF REFERENCE CODE + + Subject to the conditions below, you are granted a personal, non-exclusive, + non-transferable, revocable licence to read, study, and evaluate Reference + Code; run it on your own machine for personal, non-commercial evaluation, + study, or research; and quote or cite it with attribution. - Works cited in the documentation remain subject to their own terms. + Without prior written permission, you may not use Reference Code for a + commercial purpose, including internal use within a commercial or + governmental organisation; redistribute, publish, sublicense, sell, or host + it or any substantial portion; or create or distribute adaptations or + translations of it. -4. CONTRIBUTIONS +5. THIRD-PARTY MATERIAL AND CONTRIBUTIONS + + Files under `spec/vendor/` are third-party works under their own licences and + are not covered by either grant above. Works cited in the documentation + remain subject to their own terms. By submitting a contribution you grant the copyright holder a perpetual, worldwide, irrevocable, royalty-free licence to use, modify, and relicense - it, and you confirm you have the right to do so. You retain your own - copyright in your contribution. + it, and you confirm you have the right to do so. You retain your copyright. + +6. RETAINED RIGHTS + + No trademark rights are granted except nominative use needed for the + required attribution. No patent rights are granted. Implementing the + specification grants no certification, audit, approval, or support status. + All rights not expressly granted are reserved. + +7. TERMINATION -5. NO WARRANTY + Your rights under this licence terminate if you materially breach its + conditions and do not cure the breach within 30 days after receiving notice. + Termination affects only the breaching party. A compliant recipient retains + its own irrevocable specification implementation grant. + +8. NO WARRANTY THIS WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY ARISING FROM, OUT OF, OR IN CONNECTION WITH THIS WORK OR ITS USE. - -6. NO PATENT LICENCE - - No patent rights are granted. This is a deliberate difference from the - Apache-2.0 licence that covered earlier releases. diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 61f21f0..2e3e0e7 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -376,20 +376,29 @@ def check_publication_metadata(reporter: Reporter) -> None: license_text = read_utf8(license_path) notice_text = read_utf8(notice_path) + required_license = ( + "LLM Errata Specification Implementation and Personal Use Licence", + "commercial and non-commercial products and services", + "Implements the LLM Errata specification by Thomas Willner", + "https://github.com/thomaswillner/llm-errata", + "does not cover `prototype/`, `scripts/`, or `tests/`", + "does not imply endorsement, sponsorship, certification, or audit", + "No patent rights are granted", + "Apache License 2.0", + "irrevocable", + ) reporter.check( "license and notice", - "Personal Use Licence" in license_text + all(value in license_text for value in required_license) and "Copyright 2026 Thomas Rainer Willner" in license_text - and "Copyright 2026 Thomas Rainer Willner" in notice_text + and "Copyright 2026 Thomas Rainer Willner" in notice_text, # The Apache-2.0 grant on 0.2.0 and earlier is irrevocable. Deleting # the sentence that says so would misrepresent the rights of anyone who # already holds those releases. - and "Apache License 2.0" in license_text - and "irrevocable" in license_text, - "personal-use licence, author notice, and the irrevocable prior grant are present", - "Restore the Personal Use Licence, the Thomas Rainer Willner copyright " - "notice, and the statement that the Apache-2.0 grant on earlier releases " - "is irrevocable.", + "dual licence, product attribution, source-code boundary, author notice, and irrevocable prior grant are present", + "Restore the attributed specification implementation grant, personal-use " + "Reference Code boundary, no-endorsement and no-patent rules, Thomas " + "Rainer Willner copyright notice, and irrevocable earlier Apache grant.", ) diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index a97189e..66199d1 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -25,6 +25,41 @@ def test_unmodified_repository_passes(self) -> None: class ValidatorRejectsStructuralFaults(unittest.TestCase): + def _assert_license_mutation_is_rejected(self, old: str, new: str) -> None: + def mutate(root: Path) -> None: + path = root / "LICENSE" + text = path.read_text(encoding="utf-8") + self.assertIn(old, text, f"missing licence contract: {old}") + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("license and notice", result.stdout) + + def test_specification_implementation_grant_cannot_be_removed(self) -> None: + self._assert_license_mutation_is_rejected( + "commercial and non-commercial products and services", + "personal non-commercial experiments", + ) + + def test_required_product_attribution_cannot_be_removed(self) -> None: + self._assert_license_mutation_is_rejected( + "Implements the LLM Errata specification by Thomas Willner", + "Implements an unnamed memory specification", + ) + + def test_reference_code_cannot_be_relicensed_by_specification_grant(self) -> None: + self._assert_license_mutation_is_rejected( + "does not cover `prototype/`, `scripts/`, or `tests/`", + "also covers `prototype/`, `scripts/`, and `tests/`", + ) + + def test_false_endorsement_protection_cannot_be_removed(self) -> None: + self._assert_license_mutation_is_rejected( + "does not imply endorsement, sponsorship, certification, or audit", + "implies certification by the author", + ) + def test_missing_required_file_is_rejected(self) -> None: def mutate(root: Path) -> None: (root / "SECURITY.md").unlink() From 72740ca5645ccb8299f913036ee648c2524a6b5e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 00:14:25 +0200 Subject: [PATCH 46/88] docs: publish attributed implementation grant --- CHANGELOG.md | 6 +++++ CONTRIBUTING.md | 13 +++++++++- INDEPENDENT_IMPLEMENTATION.md | 11 +++++--- NOTICE | 11 +++++++- README.md | 19 +++++++++----- SECURITY.md | 5 ++++ scripts/validate_repo.py | 25 ++++++++++++++++++ tests/test_validate_repo.py | 48 +++++++++++++++++++++++++++++++++++ 8 files changed, 125 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbd07d2..a941247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Granted irrevocable, worldwide, royalty-free rights for independently + authored commercial and non-commercial implementations of the specification, + conditioned on accessible product attribution to LLM Errata and Thomas + Willner. Reference code remains personal-use, no patent or certification + right is granted, and the historical Apache-2.0 grant remains unchanged. + - Corrected the Inspeximus comparison after conflict-disclosed maintainer feedback and source verification. `retract_lineage` plus `rederive` is a stronger local quarantine/rebuild collision than the earlier erasure-focused diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b5aa2d..dce322c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,13 @@ Implementation contributions should include: - dependency and license information; - evidence that unrelated memory is preserved during repair. +An independently authored implementation may use the published specification, +schemas, examples, and conformance vectors under the irrevocable implementation +grant in [LICENSE](LICENSE). Product code must be authored independently rather +than copied from `prototype/`, `scripts/`, or `tests/`, and every commercial or +non-commercial product must preserve the required Thomas Willner and LLM Errata +attribution. + Never submit secrets, personal data, proprietary customer material, or confidential employer information. ## Issues and pull requests @@ -81,7 +88,11 @@ Before opening a large pull request, open an issue describing the proposed chang By submitting a contribution, you grant the copyright holder a perpetual, worldwide, irrevocable, royalty-free licence to use, modify, and relicense it, and you confirm you have the right to submit the material. You retain your own copyright. See [LICENSE](LICENSE). Cite rather than copy third-party text unless its licence clearly permits inclusion. -Note that this repository is licensed for personal use. Prior-art challenges, corrections, and design critique are welcome and need no permission. Implementing the specifications in `spec/` does require written permission, which is not unreasonably withheld for research and standards work. +Reference code in this repository remains personal-use unless separately +licensed. Prior-art challenges, corrections, and design critique need no +permission. Clean-room specification implementations need no individual grant, +but remain subject to attribution, notice-preservation, no-endorsement, and +third-party licence conditions. ## Security findings diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index 8587ebd..a66390a 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -32,7 +32,10 @@ specification version, dependencies, test commands, unsupported behavior, and licence. Shared conformance vectors are expected. Shared reference-adapter code disqualifies the implementation as independent evidence. -The current personal-use licence requires written permission to implement the -specification. Open a GitHub issue naming the research or standards scope, -repository, organization, requested version, and intended publication. A -permission grant does not imply technical endorsement. +No per-implementer permission is required for an independently authored +commercial or non-commercial implementation of the specification. The +irrevocable implementation grant requires every product or service to credit +`LLM Errata`, `Thomas Willner`, and the canonical repository in an ordinarily +accessible product location. It does not permit copying code from `prototype/`, +`scripts/`, or `tests/`, and attribution does not imply technical endorsement, +certification, or audit. See [LICENSE](LICENSE). diff --git a/NOTICE b/NOTICE index 823f959..96f6ac2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,7 +1,16 @@ LLM Errata Copyright 2026 Thomas Rainer Willner. All rights reserved. -Licensed for personal, non-commercial evaluation and study. See LICENSE. +Independent commercial and non-commercial implementations of the specification +are permitted under the attributed implementation grant in LICENSE. Reference +code remains personal-use unless separately licensed. + +Required product credit: + +Implements the LLM Errata specification by Thomas Willner — +https://github.com/thomaswillner/llm-errata + +The credit does not imply endorsement, sponsorship, certification, or audit. Version 0.2.0 and earlier were published under the Apache License 2.0. That grant is irrevocable for those releases and is not withdrawn by the current diff --git a/README.md b/README.md index ef256ed..5aafa4c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ | Status | Public concept proposal / Request for Comment | | Published | 2026-08-07 | | Research reviewed through | 2026-08-01 | -| License | Personal use only, see [LICENSE](LICENSE). Releases up to 0.2.0 were Apache-2.0. | +| License | Attributed specification implementations permitted; reference code remains personal-use. See [LICENSE](LICENSE). | This is an independent proposal. It does not represent the position of the author's employer or any organization referenced in this repository. @@ -193,12 +193,17 @@ The public research record exposes the decision criteria and evidence trail. It Copyright © 2026 Thomas Rainer Willner. -Licensed for personal, non-commercial evaluation and study. See [LICENSE](LICENSE). - -You may read, run, study, quote, and cite this work. Commercial use, -redistribution, derivative works, and implementing the specifications in -`spec/` require written permission, which is not unreasonably withheld for -research and standards work. +Commercial and non-commercial independent implementations are permitted under +an irrevocable, worldwide, royalty-free specification grant. Every product or +service implementing a material part of LLM Errata must credit **LLM Errata** +and **Thomas Willner**, with the repository URL, in an ordinarily accessible +About, Legal, documentation, acknowledgements, or NOTICE location. + +Reference code under `prototype/`, `scripts/`, and `tests/` remains licensed +only for personal, non-commercial evaluation and study unless separately +licensed. The specification grant permits clean-room implementation; it does +not permit copying reference code. Attribution does not imply endorsement, +certification, audit, or support. Version 0.2.0 and earlier were published under the Apache License 2.0. That grant is irrevocable for those releases and is not withdrawn here. diff --git a/SECURITY.md b/SECURITY.md index 56f59b4..d010920 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -50,6 +50,11 @@ Security-relevant findings include, but are not limited to: Factual disagreements, prior-art reports, and specification design proposals are not vulnerabilities; submit them through the normal contribution process. +Licence attribution does not imply security review, endorsement, or certification +by Thomas Willner. Third-party implementations have their own security and +support responsibilities; conformance or attribution alone does not place them +inside this repository's supported-version policy. + ## Operational security gate Security reporting policy is not operational-readiness evidence. G6 additionally diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 2e3e0e7..de56ddc 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -401,6 +401,31 @@ def check_publication_metadata(reporter: Reporter) -> None: "Rainer Willner copyright notice, and irrevocable earlier Apache grant.", ) + public_requirements = { + "README.md": ( + "Commercial and non-commercial independent implementations are permitted", + "Thomas Willner", + ), + "NOTICE": ("Implements the LLM Errata specification by Thomas Willner",), + "INDEPENDENT_IMPLEMENTATION.md": ("No per-implementer permission is required",), + "CONTRIBUTING.md": ("independently authored implementation",), + "SECURITY.md": ( + "Licence attribution does not imply security review, endorsement, or certification", + ), + } + public_alignment = all( + (ROOT / name).is_file() + and all(phrase in read_utf8(ROOT / name) for phrase in phrases) + for name, phrases in public_requirements.items() + ) + reporter.check( + "public licence alignment", + public_alignment, + "public documents preserve implementation rights, attribution, clean-room, and no-endorsement boundaries", + "Synchronize README, NOTICE, INDEPENDENT_IMPLEMENTATION, CONTRIBUTING, " + "and SECURITY with the dual licence contract.", + ) + def check_g2_independent_review_gate(reporter: Reporter) -> None: """Keep internal Phase 2 work from being represented as external review.""" diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 66199d1..e33cd7f 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -25,6 +25,19 @@ def test_unmodified_repository_passes(self) -> None: class ValidatorRejectsStructuralFaults(unittest.TestCase): + def _assert_public_license_mutation_is_rejected( + self, relative_path: str, old: str, new: str + ) -> None: + def mutate(root: Path) -> None: + path = root / relative_path + text = path.read_text(encoding="utf-8") + self.assertIn(old, text, f"missing public licence contract: {old}") + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("public licence alignment", result.stdout) + def _assert_license_mutation_is_rejected(self, old: str, new: str) -> None: def mutate(root: Path) -> None: path = root / "LICENSE" @@ -60,6 +73,41 @@ def test_false_endorsement_protection_cannot_be_removed(self) -> None: "implies certification by the author", ) + def test_readme_dual_license_boundary_cannot_be_removed(self) -> None: + self._assert_public_license_mutation_is_rejected( + "README.md", + "Commercial and non-commercial independent implementations are permitted", + "Only personal experiments are permitted", + ) + + def test_notice_product_attribution_cannot_be_removed(self) -> None: + self._assert_public_license_mutation_is_rejected( + "NOTICE", + "Implements the LLM Errata specification by Thomas Willner", + "Implements a memory specification", + ) + + def test_independent_implementation_permission_rule_cannot_regress(self) -> None: + self._assert_public_license_mutation_is_rejected( + "INDEPENDENT_IMPLEMENTATION.md", + "No per-implementer permission is required", + "Written permission is required", + ) + + def test_contributing_clean_room_boundary_cannot_be_removed(self) -> None: + self._assert_public_license_mutation_is_rejected( + "CONTRIBUTING.md", + "independently authored implementation", + "copy of the reference implementation", + ) + + def test_security_no_endorsement_boundary_cannot_be_removed(self) -> None: + self._assert_public_license_mutation_is_rejected( + "SECURITY.md", + "Licence attribution does not imply security review, endorsement, or certification", + "Licence attribution provides security certification", + ) + def test_missing_required_file_is_rejected(self) -> None: def mutate(root: Path) -> None: (root / "SECURITY.md").unlink() From 85cc832e10a28071f4a1cc4619eb66a558d2eca6 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 00:18:58 +0200 Subject: [PATCH 47/88] docs: separate receipt authenticity from coverage truth --- CHANGELOG.md | 4 ++++ IDEA.md | 12 +++++++++++- PRIOR_ART.md | 4 ++-- README.md | 4 ++-- RESEARCH.md | 6 +++--- REVIEW_REQUEST.md | 2 +- ROADMAP.md | 2 ++ scripts/claim_guard.py | 10 ++++++++++ spec/README.md | 6 ++++++ tests/test_claim_guard.py | 27 +++++++++++++++++++++++++++ 10 files changed, 68 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a941247..ca45d55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ Changes to the bounded novelty statement or the source comparison are recorded h conditioned on accessible product attribution to LLM Errata and Thomas Willner. Reference code remains personal-use, no patent or certification right is granted, and the historical Apache-2.0 grant remains unchanged. +- Split the callback requirement into independently evaluated authenticity and + coverage-truthfulness properties. A valid signature authenticates the + importer and receipt bytes; it cannot turn overstated, missing, or opaque + coverage into a truthful claim. - Corrected the Inspeximus comparison after conflict-disclosed maintainer feedback and source verification. `retract_lineage` plus `rederive` is a diff --git a/IDEA.md b/IDEA.md index 1eed14c..3e8e00a 100644 --- a/IDEA.md +++ b/IDEA.md @@ -248,7 +248,17 @@ The search deliberately broadened, collided, narrowed, and repeated. These adjac | Cross-recipient lifecycle notification | [vCon Lifecycle using SCITT](https://datatracker.ietf.org/doc/html/draft-howe-vcon-lifecycle-01), [SCITT architecture](https://datatracker.ietf.org/doc/rfc9943/), [IAB deletion framework](https://github.com/InteractiveAdvertisingBureau/Data-Subject-Rights/blob/main/Data%20Deletion%20Request%20Framework.md) | This is the strongest formal standards/control-plane collision. The individual vCon draft defines sent/received events, amendments, consent changes, deletion or expiry notifications, and recipient acknowledgments backed by SCITT inclusion receipts. Inclusion or acknowledgment does not establish repair of AI summaries, vectors, graphs, and behavior. | | Revocation and provenance updates | [W3C credential status and refresh](https://www.w3.org/TR/vc-data-model-2.0/#refreshing), [W3C Bitstring Status List](https://www.w3.org/TR/vc-bitstring-status-list/), [C2PA 2.4](https://spec.c2pa.org/specifications/specifications/2.4/specs/C2PA_Specification.html) | Exported objects can expose status, refresh, revocation, and signed update manifests. They do not impose AI-memory-specific descendant repair or the repair triad. | | Derived-data deletion | [Palantir US10956406B2](https://patents.google.com/patent/US10956406B2/en) and long-term-memory security research ([survey](https://arxiv.org/html/2604.16548v1)) | Rebuilding derived data after source deletion and testing for reappearance are not new. The candidate is the interoperable semantic conformance profile, not either primitive. | -| **Remaining feature conjunction** | **No exact public implementation found in the reviewed sources required all four together** | **Post-export update delivery; importer-side quarantine and repair of the known descendant closure; negative, positive, and preservation tests; and a signed, coverage-aware callback bound to the erratum and pre/post state.** | +| **Remaining feature conjunction** | **No exact public implementation found in the reviewed sources required all four together** | **Post-export update delivery; importer-side quarantine and repair of the known descendant closure; negative, positive, and preservation tests; and an authenticated, coverage-truthful callback bound to the erratum and pre/post state.** | + +The callback requirement contains two independent properties. **D1, +authenticity:** a valid signature binds the importer to the event, pre/post +state, stores, aggregate, and limitations. **D2, coverage truthfulness:** those +signed coverage claims match the declared required scope and do not upgrade +missing, opaque, skipped, or failed evidence. + +Signature validity authenticates the importer and receipt bytes; it does not establish that the reported coverage is truthful. + +Coverage truthfulness requires the signed stores, aggregate, and limitations to match the declared required scope without upgrading missing or opaque evidence. The broad candidates were rejected: “a personal LLM wiki,” “portable memory,” “a memory makefile,” “verified forgetting,” “a temporal contradiction ledger,” and “self-correcting memory.” A signed post-export update feed was also rejected: EngramSpec already provides cross-runtime corrections and incremental diffs, while vCon Lifecycle/SCITT and downstream-deletion frameworks cover event and acknowledgment control planes. The surviving combination is still a synthesis of known mechanisms, so its strongest defensible claim is product and conformance design—not fundamental invention. diff --git a/PRIOR_ART.md b/PRIOR_ART.md index 849d1cb..674bb15 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -15,7 +15,7 @@ The proposal survives only if the following four requirements are evaluated toge | A | **Post-export update delivery:** an authenticated correction, supersession, or erasure reaches an importer after the original memory transfer. | | B | **Descendant quarantine and repair:** the importer blocks the root and its known local derivation closure before recall, then retires or rebuilds affected descendants. | | C | **Repair triad:** conformance tests the negative, positive when applicable, and preservation postconditions. | -| D | **Coverage-aware callback:** the importer returns a signed result bound to the erratum and pre-/post-repair state, including incomplete or unknown store coverage. | +| D | **Authenticated, coverage-truthful callback:** D1 authenticates the importer and signed receipt bytes; D2 requires the signed stores, aggregate, and limitations to match the declared required scope, including incomplete or unknown coverage. | No individual mechanism is claimed as new. @@ -147,7 +147,7 @@ A complete collision should identify one dated public implementation or normativ 2. a valid event causes quarantine of the root and the importer's known local descendant closure before recall; 3. descendants are rebuilt or retired without destroying unrelated retained memory; 4. conformance requires negative, positive when applicable, and preservation tests; -5. the importer returns a signed callback bound to the erratum and pre-/post-repair state with explicit incomplete or unknown coverage. +5. the importer returns an authenticated callback bound to the erratum and pre-/post-repair state, while a separate coverage-truthfulness check requires explicit incomplete or unknown coverage. When such evidence appears, this repository should record it, narrow or withdraw the novelty statement, and preserve the correction in [CHANGELOG.md](CHANGELOG.md). The correct response to a complete collision is not semantic argument—it is an erratum to LLM Errata itself. diff --git a/README.md b/README.md index 5aafa4c..b5c1162 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ observe → quarantine → rebuild → test → attest 2. **Quarantine:** block that root and its known local descendants before further recall. 3. **Rebuild:** retire invalid artifacts and reconstruct mixed artifacts from still-valid inputs. 4. **Test:** run the repair triad across the declared stores and behavioral scope. -5. **Attest:** return a signed, coverage-aware receipt bound to the erratum and the importer's pre-repair and post-repair state. +5. **Attest:** return an authenticated, coverage-truthful receipt bound to the erratum and the importer's pre-repair and post-repair state. Its signature authenticates the bytes; separate coverage rules determine whether the signed claim is honest. An importer that cannot inspect a relevant cache or derived store reports `unknown`. It does not silently turn incomplete coverage into success. @@ -85,7 +85,7 @@ This repository does **not** claim to invent portable memory, correction feeds, The narrow research conclusion is: -> **No exact public implementation found in the reviewed sources required all four together: post-export update delivery; importer-side quarantine and repair of the known descendant closure; negative, positive, and preservation tests; and a signed, coverage-aware callback bound to the erratum and pre/post state.** +> **No exact public implementation found in the reviewed sources required all four together: post-export update delivery; importer-side quarantine and repair of the known descendant closure; negative, positive, and preservation tests; and an authenticated, coverage-truthful callback bound to the erratum and pre/post state.** This is a novel synthesis with an apparently unimplemented conformance gap. It is not a claim of patentability, a “world first,” or freedom to operate. [EngramSpec](https://engramspec.org/) is the strongest AI-memory transport collision. The individual [vCon Lifecycle using SCITT draft](https://datatracker.ietf.org/doc/html/draft-howe-vcon-lifecycle-01) is the strongest formal standards/control-plane collision. [Shomei](https://shomei.ai/docs/governance-and-receipts/) and [Inspeximus](https://github.com/DanceNitra/inspeximus) are the closest local governance and correction collisions. diff --git a/RESEARCH.md b/RESEARCH.md index 4b591fe..dfd63e5 100644 --- a/RESEARCH.md +++ b/RESEARCH.md @@ -16,7 +16,7 @@ The broad question was: The question that survived screening was narrower: -> As of 2026-08-01, did any reviewed public implementation or normative profile require an importer of exported AI memory to receive later errata, quarantine and repair the memory's locally derived descendants, verify the repair behaviorally, and return a signed, coverage-aware result to the root owner? +> As of 2026-08-01, did any reviewed public implementation or normative profile require an importer of exported AI memory to receive later errata, quarantine and repair the memory's locally derived descendants, verify the repair behaviorally, and return an authenticated, coverage-truthful result to the root owner? The screened result is **LLM Errata**: @@ -27,7 +27,7 @@ The exact surviving feature conjunction is: 1. **Post-export update delivery.** A previously exported memory root remains addressable through an authenticated, monotonically sequenced correction, supersession, or erasure channel. 2. **Importer-side descendant quarantine and repair.** After validating an erratum, the importer blocks the root and its known local derivation closure before recall, then retires or rebuilds affected summaries, embeddings, profile fields, graph nodes, caches, and downstream exports from still-valid inputs. 3. **Three-way behavioral verification.** The importer tests that the retired belief is absent, the replacement is active when applicable, and unrelated memory survives. LLM Errata calls these the negative, positive, and preservation checks. -4. **Signed, coverage-aware callback.** The importer returns a receipt bound to the erratum and pre-/post-repair state roots, identifies the stores inspected, reports surfaces it cannot observe as `unknown`, and cannot claim aggregate success while a required store remains unresolved. +4. **Authenticated, coverage-truthful callback.** The importer returns a signed receipt bound to the erratum and pre-/post-repair state roots, identifies the stores inspected, reports surfaces it cannot observe as `unknown`, and cannot claim aggregate success while a required store remains unresolved. Signature verification authenticates the importer and bytes; coverage evaluation separately tests whether the signed stores, aggregate, and limitations truthfully describe the required scope. The claim is about the **complete conjunction**. None of its individual mechanisms is claimed as new. @@ -190,7 +190,7 @@ The prior-art conclusion should be revised or withdrawn if a dated public source 2. A valid event causes pre-repair quarantine and traversal of the importer's local descendant closure across relevant stores. 3. Mixed descendants are rebuilt or retired while unrelated memory is preserved. 4. Conformance requires negative, positive when applicable, and preservation checks. -5. The importer returns a signed callback bound to the triggering event and pre-/post-repair state, with explicit `partial`, `unknown`, or `failed` coverage where full verification is impossible. +5. The importer returns an authenticated callback bound to the triggering event and pre-/post-repair state, with separately evaluated coverage truthfulness and explicit `partial`, `unknown`, or `failed` results where full verification is impossible. Partial matches should be added to [PRIOR_ART.md](PRIOR_ART.md), not concealed. A future implementation that closes the whole gap would not make the user problem disappear; it would invalidate the claim that the conformance boundary remained unimplemented. diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md index 972e994..b09c303 100644 --- a/REVIEW_REQUEST.md +++ b/REVIEW_REQUEST.md @@ -18,7 +18,7 @@ any earlier public implementation or normative profile that requires all of: 1. post-export update delivery to prior importers; 2. importer-side quarantine and repair of known local descendants; 3. negative, positive, and preservation verification; and -4. a signed, coverage-aware callback bound to the event and importer state. +4. an authenticated, coverage-truthful callback bound to the event and importer state: D1 verifies signer and byte binding; D2 checks that signed stores, aggregate, and limitations truthfully represent the declared required scope. Partial collisions are valuable and should narrow the proposal. diff --git a/ROADMAP.md b/ROADMAP.md index d3787c6..79e2188 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -95,6 +95,8 @@ The opaque adapter is not a toy edge case. It is the control that proves the agg - a stale export cannot silently restore the retired state; - an `unknown`, `partial`, or `failed` required store prevents aggregate success; - receipts bind the erratum plus deterministic pre-repair and post-repair state roots; +- signature verification authenticates receipt bytes but cannot certify coverage truthfulness; +- signed stores, aggregate, and limitations match the declared required scope without upgrading missing or opaque evidence; - tests run without an API key; model-graded probes are optional and clearly separated; - the complete demo is reproducible from a clean checkout. diff --git a/scripts/claim_guard.py b/scripts/claim_guard.py index 22456c6..56a6f69 100644 --- a/scripts/claim_guard.py +++ b/scripts/claim_guard.py @@ -82,6 +82,16 @@ "Aggregate success requires every required store to be `verified`.", "no green aggregate over unknown coverage", ), + ( + "IDEA.md", + "Signature validity authenticates the importer and receipt bytes; it does not establish that the reported coverage is truthful.", + "signature authenticity boundary", + ), + ( + "IDEA.md", + "Coverage truthfulness requires the signed stores, aggregate, and limitations to match the declared required scope without upgrading missing or opaque evidence.", + "coverage truthfulness boundary", + ), ( "PRIOR_ART.md", "## What would invalidate the claim", diff --git a/spec/README.md b/spec/README.md index d1a5f6f..2fbb9f1 100644 --- a/spec/README.md +++ b/spec/README.md @@ -9,6 +9,12 @@ than this one can be built and checked against the same contract. | `receipt.schema.json` | What an importer returns after acting on one. | | `vectors/manifest.json` | Every vector, which schema it belongs to, whether it must validate, and — for an invalid vector — the text that must appear in the rejection. | | `vendor/json-schema-test-suite/` | Cases from the official JSON-Schema-Test-Suite, vendored unmodified. | + +Receipt conformance evaluates two independent properties. D1 authenticity +verifies the importer and every signable byte. D2 coverage truthfulness checks +that signed stores, aggregate, and limitations match the declared required +scope. A correctly signed receipt that overstates coverage passes D1 and fails +D2; signature validity never upgrades missing or opaque evidence. | `semantic/probes.json` | Named, strict semantic-probe sets for offline conformance. | | `semantic/verifier-config.json` | Exact synthetic verifier configuration, whose canonical digest binds every observation. | | `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | diff --git a/tests/test_claim_guard.py b/tests/test_claim_guard.py index 58c6e5f..327dd3d 100644 --- a/tests/test_claim_guard.py +++ b/tests/test_claim_guard.py @@ -78,6 +78,33 @@ def mutate(root: Path) -> None: class ClaimGuardRejectsInvariantLoss(unittest.TestCase): + def test_signature_authenticity_cannot_be_misstated_as_coverage_truth(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "IDEA.md", + "Signature validity authenticates the importer and receipt bytes; " + "it does not establish that the reported coverage is truthful.", + "Signature validity proves that the reported coverage is truthful.", + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("signature authenticity boundary", result.stdout) + + def test_coverage_truthfulness_cannot_upgrade_missing_evidence(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "IDEA.md", + "Coverage truthfulness requires the signed stores, aggregate, and " + "limitations to match the declared required scope without upgrading " + "missing or opaque evidence.", + "Coverage truthfulness permits missing and opaque evidence to be upgraded.", + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("coverage truthfulness boundary", result.stdout) + def test_inverted_loop_order_is_rejected(self) -> None: def mutate(root: Path) -> None: path = root / "IDEA.md" From 08b95263c9ed700c43aea0b285696956cc23e878 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 00:20:06 +0200 Subject: [PATCH 48/88] docs: foreground cryptographic refusal evidence --- CHANGELOG.md | 4 ++++ docs/CRYPTOGRAPHY_QUALIFICATION.md | 23 ++++++++++++++++++++++- tests/test_readiness.py | 16 ++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca45d55..20e96d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ Changes to the bounded novelty statement or the source comparison are recorded h coverage-truthfulness properties. A valid signature authenticates the importer and receipt bytes; it cannot turn overstated, missing, or opaque coverage into a truthful claim. +- Reordered the production-cryptography qualification to foreground observed + refusal of malformed lengths, non-canonical scalars, tampered messages and + signatures, and wrong keys. Successful RFC vectors remain compatibility + evidence and do not establish constant-time or production assurance. - Corrected the Inspeximus comparison after conflict-disclosed maintainer feedback and source verification. `retract_lineage` plus `rederive` is a diff --git a/docs/CRYPTOGRAPHY_QUALIFICATION.md b/docs/CRYPTOGRAPHY_QUALIFICATION.md index 2856760..800ee63 100644 --- a/docs/CRYPTOGRAPHY_QUALIFICATION.md +++ b/docs/CRYPTOGRAPHY_QUALIFICATION.md @@ -31,6 +31,25 @@ checked RFC 8032 vectors, but Python big-integer operations are not a suitable side-channel boundary for production signing. The `Signer` protocol in `prototype/signing.py` isolates replacement from controller and receipt logic. +### Verification refusal evidence + +The stronger security-relevant reference evidence is not that five valid +signatures succeed; it is that invalid inputs fail closed. The checked suite +demonstrates refusal of: + +- a non-canonical scalar (`S >= L`), preventing signature malleability; +- malformed public-key and signature lengths, returning `False` rather than + raising into an ambiguous caller path; +- a tampered message; +- a tampered signature; and +- a wrong public key. + +The suite also refuses malformed signature strings at the `VerificationKey` +seam and refuses a seed of the wrong length. These are observed reference-code +properties in `tests/test_ed25519.py`, not claims about a future production +binding. They do not prove constant-time behavior, resistance to side channels, +safe key lifecycle, current-library audit status, or G3. + ### PyCA `cryptography` | Item | Observed evidence | @@ -114,7 +133,9 @@ not independent security evidence. The compatibility probe loaded the five RFC 8032 vectors already published in `tests/test_ed25519.py`, constructed PyCA private keys from each 32-byte seed, and compared raw public keys and signatures with literal expected values. It -then verified every signature with the corresponding raw public key. +then verified every signature with the corresponding raw public key. This is a +wire-compatibility oracle; the refusal-path tests above are the stronger +security-relevant evidence about the current reference verifier. Observed result: diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 8ee4aa8..b33df36 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -37,6 +37,22 @@ class ReadinessCheckerPasses(unittest.TestCase): + def test_cryptography_qualification_foregrounds_refusal_evidence(self) -> None: + qualification = ( + Path(__file__).resolve().parents[1] + / "docs" + / "CRYPTOGRAPHY_QUALIFICATION.md" + ).read_text(encoding="utf-8") + for phrase in ( + "non-canonical scalar", + "malformed public-key and signature lengths", + "tampered message", + "tampered signature", + "wrong public key", + ): + with self.subTest(phrase=phrase): + self.assertIn(phrase, qualification) + def test_current_not_ready_ledger_is_honest(self) -> None: with repo_copy() as root: result = run_checker(root, SCRIPT) From 1df7355d4a111537c491d08f596868b6fa520cd0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 01:26:21 +0200 Subject: [PATCH 49/88] docs: publish corrected attributed review target --- docs/PUBLICATION_LOG.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 6b03463..263c126 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -1,7 +1,7 @@ # Publication log This log records publication and outreach evidence. It does not upgrade a -readiness gate. Last verified: 2026-08-12. +readiness gate. Last verified: 2026-08-13. ## GitHub public calls @@ -22,10 +22,13 @@ readiness gate. Last verified: 2026-08-12. | Independent implementation call | [Issue #5](https://github.com/thomaswillner/llm-errata/issues/5) and its [implementation update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246) | Open call for independently authored adapters and a separately produced validator. | | Phase 3 system nominations | [Issue #6](https://github.com/thomaswillner/llm-errata/issues/6) and its [nomination update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440) | Open nominations only; no experiment is authorized or underway. | | GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | -| Complete Phase 2 review target | Commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2), digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` | Internal Phase 2 surface is complete and locally validated. Published for independent review; publication and internal tests do not satisfy G2. | +| Historical Phase 2 review target | Commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2), digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` | Stale: this target predated the accepted prior-art corrections and must not be used for a current review. Preserved as historical publication evidence. | +| Corrected complete Phase 2 review target | Commit [`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878), digest `03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14` | Complete corrected specification, prior-art, licensing, and cryptographic-qualification surface. Checkout and committed-source digests match; exact-target CI passed Python 3.11 and 3.13. Internal evidence does not satisfy G2. | | Final independent-implementation call | [Issue #5 completion update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5272475120) | Publishes the immutable Phase 2 commit and digest for two independent adapters and a separate validator. No implementation has been accepted; G4 remains `BLOCKED`. | | Final three-system nomination call | [Issue #6 completion update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5272475614) | Publishes the immutable Phase 2 target and exact nomination fields. No system authorization or experiment evidence exists; G5 remains `BLOCKED`. | | Interested-party review follow-up | [Inspeximus maintainer reply](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5272476094) | Acknowledges the source-grounded correction and points to the final review target. Conflict disclosure remains controlling; a future review from this maintainer cannot satisfy G2 alone. | +| Stale-target finding | [Inspeximus maintainer correction](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5273206921) | Correctly found that the prior frozen commit predated the accepted prior-art corrections. This is interested-party feedback, not independent G2 evidence; the finding required a new immutable target. | +| Corrected maintainer reply | [Human reply and re-frozen target](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5273960394) | Acknowledges the publication defect, publishes corrected commit and digest, answers the licence question, and preserves the conflict boundary. DanceNitra's offered review remains interested-party evidence and requires a separate disinterested reviewer. | | Phase 2 completion announcement | [Discussion #9 completion update](https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648) | Announces internal completion and routes falsifiers to Issues #4, #5, #6, and #10. Publication only; it records no external acceptance or readiness-gate change. | | Six-step readiness checkpoint | [PR #8 checkpoint](https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272526975) | Audits every approved production-readiness step against exact internal and external evidence. Records `NOT_PROD_READY`, the external blockers, PR #3 status, validation, and publication boundaries without claiming certification. | @@ -34,12 +37,18 @@ it is not an acceptance of an independent reviewer role and does not satisfy G2. No adapter, validator, operated-system approval, or CODEOWNER volunteer has accepted. -The complete conformance review target is source commit -[`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2) +The current complete conformance review target is source commit +[`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878) with canonical Phase 2 surface digest -`9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804`. +`03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14`. The digest was recomputed both from the checkout and from `git show` at that -commit with an exact match. Earlier source commit +commit with an exact match. The exact-target [GitHub Actions run +31646616085](https://github.com/thomaswillner/llm-errata/actions/runs/31646616085) +passed Python 3.11 and Python 3.13. The previously frozen source commit +[`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2) +and digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` +are historical and stale because they predate the accepted prior-art +corrections. Earlier source commit [`35052868cde8b6f16fc6c20b4136ae0277f9a00e`](https://github.com/thomaswillner/llm-errata/commit/35052868cde8b6f16fc6c20b4136ae0277f9a00e) and digest `a3133f759620ed942d6cd519d303c0be1554f0193b89e0a141c410aaf4c047b9` remain historical pre-completion checkpoints, not current review targets. @@ -97,6 +106,6 @@ external evidence. Phase 2 is internally complete: explicit quarantine checkpoints, key rotation, concurrent conflicts, invalid targets, confidentiality, semantic probes, and every-field receipt binding are covered. G2 remains `BLOCKED` pending a dated independent external review of exact commit -`50e895fbfec544b16c94caa07bf2d1f4049a42e2` and digest -`9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804`. +`08b95263c9ed700c43aea0b285696956cc23e878` and digest +`03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14`. Repository verdict remains `NOT_PROD_READY`. From 5d4fa7b06f720201b460b5cf545050406fe119e1 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 02:31:06 +0200 Subject: [PATCH 50/88] docs: record targeted production review outreach --- .learnings/LEARNINGS.md | 45 +++++++++++++++++++++++++++++++++++++++++ docs/PUBLICATION_LOG.md | 15 +++++++++----- 2 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 .learnings/LEARNINGS.md diff --git a/.learnings/LEARNINGS.md b/.learnings/LEARNINGS.md new file mode 100644 index 0000000..9ba1c47 --- /dev/null +++ b/.learnings/LEARNINGS.md @@ -0,0 +1,45 @@ +# Project learnings + +## [LRN-20260813-001] best_practice + +**Logged**: 2026-08-13T02:28:16+02:00 +**Priority**: high +**Status**: resolved +**Area**: docs + +### Summary + +Verify every pinned GitHub blob link against the exact commit before publishing outreach. + +### Details + +The targeted implementation draft initially linked to `IMPLEMENTATION_CALL.md` +at the corrected frozen commit, but that path does not exist in the tree. +Pre-publication `git cat-file -e :` validation caught the defect +before any external comment was written. Full link validation then found the +same dead path in an older ignored draft, showing that publication artifacts +must be checked even when they are not tracked. + +### Suggested Action + +For every external comment containing a pinned blob URL, extract its commit and +path, verify the object with `git cat-file -e`, then confirm public HTTP +resolution before posting. + +### Metadata + +- Source: error +- Related Files: `.scratch/g2-publication/issue-5-targeted-implementation.md` +- Tags: github, outreach, immutable-target, link-validation +- Pattern-Key: publication.verify_pinned_blob_before_post +- Recurrence-Count: 1 +- First-Seen: 2026-08-13 +- Last-Seen: 2026-08-13 + +### Resolution + +- **Resolved**: 2026-08-13T02:28:16+02:00 +- **Commit/PR**: agent/g2-publication publication follow-up +- **Notes**: Replaced nonexistent blob URLs with Issue #5 links and verified all remaining pinned paths before publication. + +--- diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 263c126..dcb75c8 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -8,7 +8,7 @@ readiness gate. Last verified: 2026-08-13. | Surface | Public artifact | Status | |---|---|---| | Draft implementation and review surface | [PR #8](https://github.com/thomaswillner/llm-errata/pull/8) | Open draft, targeting `agent/prod-readiness`. | -| Independent review request | [Issue #4](https://github.com/thomaswillner/llm-errata/issues/4) and its [exact review target](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269378077) | Open call for conformance, novelty, security, and distributed-systems review. | +| Independent review request | [Issue #4](https://github.com/thomaswillner/llm-errata/issues/4) and its [corrected targeted review request](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5274367774) | Open call against corrected immutable target. Portable Agent Memory and libsodium maintainers were asked for distinct, bounded review or referral roles. Invitation only; G2 and G3 remain `BLOCKED`. | | Targeted independent-review recruitment | [Inspeximus maintainer invitation](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269444341) | Invitation to challenge the stated collision boundary; it is not external evidence. | | Interested-party prior-art feedback | [Inspeximus maintainer response](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5271988057) | Conflict-disclosed preliminary feedback. Source verification confirmed `retract_lineage` as a stronger local quarantine/rebuild collision and narrowed the coverage comparison. The disclosed dogfood ratio was not independently reproduced. Maintainer offered a future scoped review, but explicitly does not meet G2 independence alone. | | Targeted adapter recruitment | [Remnic invitation](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269523953) | Invitation to become a future independently authored adapter after Phase 2 completion; it is not technical evidence. | @@ -29,13 +29,18 @@ readiness gate. Last verified: 2026-08-13. | Interested-party review follow-up | [Inspeximus maintainer reply](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5272476094) | Acknowledges the source-grounded correction and points to the final review target. Conflict disclosure remains controlling; a future review from this maintainer cannot satisfy G2 alone. | | Stale-target finding | [Inspeximus maintainer correction](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5273206921) | Correctly found that the prior frozen commit predated the accepted prior-art corrections. This is interested-party feedback, not independent G2 evidence; the finding required a new immutable target. | | Corrected maintainer reply | [Human reply and re-frozen target](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5273960394) | Acknowledges the publication defect, publishes corrected commit and digest, answers the licence question, and preserves the conflict boundary. DanceNitra's offered review remains interested-party evidence and requires a separate disinterested reviewer. | +| Corrected adapter and validator recruitment | [Issue #5 targeted follow-up](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5274369945) | Supersedes obsolete target and permission language; asks the Remnic maintainer about one clean-room adapter and the ai-memory-mcp maintainer about the separate validator role. No acceptance or implementation evidence is recorded. | +| Corrected operated-system recruitment | [Issue #6 targeted follow-up](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5274371747) | Supersedes obsolete target; asks Mem0 and Cognee maintainers for exact-version operator nominations or impracticality findings. No experiment or account action is authorized. | +| Corrected PR checkpoint | [PR #8 correction](https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5274373337) | Marks prior PR comments' target and digest historical, publishes corrected immutable target and routes readers to the three current evidence calls. No user mention was added. | +| Corrected Discussion announcement | [Discussion #9 follow-up](https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17995307) | Publishes corrected target, current implementation grant, and current external-evidence calls without mentioning additional users. Publication only; readiness remains unchanged. | | Phase 2 completion announcement | [Discussion #9 completion update](https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648) | Announces internal completion and routes falsifiers to Issues #4, #5, #6, and #10. Publication only; it records no external acceptance or readiness-gate change. | | Six-step readiness checkpoint | [PR #8 checkpoint](https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272526975) | Audits every approved production-readiness step against exact internal and external evidence. Records `NOT_PROD_READY`, the external blockers, PR #3 status, validation, and publication boundaries without claiming certification. | -One targeted invitation now has the interested-party response recorded above; -it is not an acceptance of an independent reviewer role and does not satisfy -G2. No adapter, validator, operated-system approval, or CODEOWNER volunteer has -accepted. +One targeted invitation has the interested-party response recorded above; it +is not an acceptance of an independent reviewer role and does not satisfy G2. +The corrected follow-ups use six role-specific mentions across three issues and +no mentions in PR #8 or Discussion #9. No response to those new follow-ups, +adapter, validator, operated-system approval, or CODEOWNER volunteer is recorded. The current complete conformance review target is source commit [`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878) From bf8950270a9009a553c0a2b24a32c9ab0f1eb209 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 02:35:37 +0200 Subject: [PATCH 51/88] docs: design active publication surface guard --- CONTEXT.md | 36 ++++++ ...01-append-only-publication-supersession.md | 27 ++++ ...08-13-active-publication-surface-design.md | 120 ++++++++++++++++++ 3 files changed, 183 insertions(+) create mode 100644 CONTEXT.md create mode 100644 docs/adr/0001-append-only-publication-supersession.md create mode 100644 docs/superpowers/specs/2026-08-13-active-publication-surface-design.md diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..44281ee --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,36 @@ +# LLM Errata Publication Context + +This context defines the language used to distinguish current public instructions +from retained publication history without weakening evidence provenance. + +## Language + +**Active surface**: +A tracked public artifact that currently instructs a reviewer, implementer, +validator, or system operator which immutable target and evidence role to use. +_Avoid_: Latest comment, current main, final link + +**Historical surface**: +A retained public artifact that accurately records an earlier project state but +must not be used as current instructions. +_Avoid_: Bad comment, deleted record + +**Frozen review target**: +An immutable source commit paired with the canonical G2 surface digest that a +review or independent implementation must bind. +_Avoid_: Branch head, latest build + +**Supersession**: +An append-only correction that explicitly marks an earlier public instruction +historical and publishes its current replacement. +_Avoid_: Rewrite, deletion + +**Evidence role**: +One independently attributable responsibility in the readiness program, such as +reviewer, adapter author, validator author, or operated-system owner. +_Avoid_: Contributor, interested party + +**Recruitment evidence**: +Proof that a bounded request was published to a relevant party. It is not proof +that the party accepted, performed, or passed the requested work. +_Avoid_: External evidence, review evidence diff --git a/docs/adr/0001-append-only-publication-supersession.md b/docs/adr/0001-append-only-publication-supersession.md new file mode 100644 index 0000000..10b7a7c --- /dev/null +++ b/docs/adr/0001-append-only-publication-supersession.md @@ -0,0 +1,27 @@ +--- +status: accepted +--- + +# Keep GitHub history append-only and track current instructions separately + +GitHub comments remain immutable publication history because editing or deleting +them would weaken provenance. A tracked active-surface manifest identifies the +current frozen target, licence posture, evidence roles, and superseding URLs; +deterministic repository checks validate that manifest, while live GitHub checks +remain a publication-time gate rather than part of offline `make check`. + +## Considered options + +- Editing old comments was rejected because it obscures what reviewers actually + saw and GitHub exposes edit history anyway. +- Treating the newest comment as current was rejected because chronology does + not prove semantic supersession or role completeness. +- Scraping GitHub during every repository check was rejected because network, + authentication, rate limits, and mutable external state make builds + nondeterministic. + +## Consequences + +Every current public call must appear once in the manifest and every replacement +must name its superseded surface. Historical URLs remain permitted only outside +the active manifest and must be clearly labelled as historical where presented. diff --git a/docs/superpowers/specs/2026-08-13-active-publication-surface-design.md b/docs/superpowers/specs/2026-08-13-active-publication-surface-design.md new file mode 100644 index 0000000..67b7994 --- /dev/null +++ b/docs/superpowers/specs/2026-08-13-active-publication-surface-design.md @@ -0,0 +1,120 @@ +# Active Publication Surface Reconciliation Design + +## Purpose + +Prevent reviewers, independent implementers, validator authors, and operated- +system owners from following obsolete LLM Errata targets or licence instructions. +The repository will distinguish current public instructions from append-only +GitHub history and fail deterministic checks when the current contract drifts. + +This hardens recruitment and publication integrity. It does not satisfy an +external readiness gate and cannot change `NOT_PROD_READY` by itself. + +## Selected approach + +Use a tracked active-surface manifest plus a dependency-free offline validator. +The manifest is the sole repository authority for which public artifacts are +current. Historical comments remain untouched and visible; current comments +explicitly supersede them. + +The rejected alternatives are: + +1. edit or delete obsolete GitHub comments, which weakens provenance; +2. infer current instructions from the newest comment, which confuses chronology + with semantic authority; and +3. query GitHub from `make check`, which makes deterministic validation depend on + authentication, rate limits, and mutable network state. + +## Active-surface manifest + +Create `publication/active-surfaces.json` with: + +- `schema_version`: exactly `1`; +- `review_target.commit`: exact 40-character lowercase Git object ID; +- `review_target.surface_digest`: exact 64-character lowercase SHA-256; +- `license`: specification implementation posture, mandatory attribution, and + personal-use reference-code boundary; +- `surfaces`: one object for each active Issue #4, Issue #5, Issue #6, PR #8, and + Discussion #9 follow-up; +- each surface: stable ID, kind, URL, publication date, gates, evidence roles, + mentioned GitHub identities, superseded URLs, and evidence boundary. + +The manifest will point to source target +`08b95263c9ed700c43aea0b285696956cc23e878` and digest +`03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14`. + +## Deterministic validator + +Create `scripts/check_publication.py`, using only Python standard library. It +must reject: + +- malformed schema, commit, digest, URL, gate, date, role, or mention fields; +- duplicate active IDs, URLs, evidence roles, or GitHub mentions; +- a current surface that does not bind the canonical commit and digest; +- obsolete target `50e895fbfec544b16c94caa07bf2d1f4049a42e2` or digest + `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804`; +- obsolete case-by-case implementation permission language; +- a commercial/non-commercial implementation grant without LLM Errata, Thomas + Willner, repository, and personal-use reference-code boundaries; +- a surface that represents invitation, maintainer work, CI, or publication as + independent readiness evidence; +- missing current surfaces for G2/G3 review, G4 adapter/validator work, G5 + operated systems, PR correction, or Discussion correction. + +Successful validation means the tracked publication contract is internally +consistent. It does not prove URLs are live, mentions notified users, conflicts +are complete, or independent work occurred. + +## Live publication gate + +Network-dependent publication checkpoints continue to verify: + +- every target identity maps to the cited project and role; +- every pinned blob path exists at its exact commit before posting; +- public HTTP URLs resolve; +- no duplicate follow-up exists before mutation; +- rendered comment body, author, URL, and mention count match the admitted draft; +- external responses are read before any retry or further mention. + +These observations belong in `docs/PUBLICATION_LOG.md` and durable loop state, +not in the offline validator's pass condition. + +## Integration + +- Add `publication/active-surfaces.json` and `scripts/check_publication.py` to + required repository files. +- Add `make publication`; run it from `make check` before unit tests. +- Add `.github/workflows/validate.yml` execution through existing `make check`. +- Add negative tests in `tests/test_publication.py`; every advertised rejection + must be demonstrated by a mutation that fails for the intended reason. +- Update `AGENTS.md`, `CONTRIBUTING.md`, and `docs/PUBLICATION_STRATEGY.md` with + the active/historical boundary and pre-post pinned-link check. + +## Failure handling + +- Structural or semantic manifest defects exit `1` and block commit/publication. +- Missing or unreadable manifest exits `2` as inconclusive, never pass. +- Live GitHub ambiguity triggers read-after-write reconciliation before retry. +- A new external response pauses publication mutation until its recommendations, + conflicts, and gate eligibility are classified. + +## Verification + +Required gates: + +1. focused publication tests demonstrate red, then green; +2. `python3 scripts/check_publication.py` passes current manifest; +3. `make check` passes all repository, claim, readiness, publication, unit, and + demo checks; +4. `make links` resolves all citations under repository policy; +5. `git diff --check` passes; +6. branch push produces Python 3.11 and 3.13 exact-head CI success; +7. readiness verdict remains `NOT_PROD_READY` unless independent evidence meets + every gate contract. + +## Scope boundary + +This slice does not create an independent reviewer, implementation, validator, +system operator, production signer, or measured operational report. It improves +the probability and integrity of obtaining that evidence; G2 through G6 remain +blocked until qualifying external work is recorded. From 00b8b5f8c3e4b2f9ba257fb208c2fe6fdd5bf880 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 02:37:16 +0200 Subject: [PATCH 52/88] docs: plan active publication surface guard --- ...-08-13-active-publication-surface-guard.md | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-13-active-publication-surface-guard.md diff --git a/docs/superpowers/plans/2026-08-13-active-publication-surface-guard.md b/docs/superpowers/plans/2026-08-13-active-publication-surface-guard.md new file mode 100644 index 0000000..4c4d532 --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-active-publication-surface-guard.md @@ -0,0 +1,204 @@ +# Active Publication Surface Guard Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make stale public review targets and obsolete implementation-permission language fail deterministic repository checks while preserving append-only GitHub history. + +**Architecture:** A tracked JSON manifest records the five active public surfaces, their immutable source target, evidence roles, mentions, superseded URLs, licence posture, and evidence boundary. A dependency-free Python checker validates exact current values offline; live GitHub identity, rendering, and liveness remain publication-time checks recorded in the publication log. + +**Tech Stack:** Python 3.11/3.13 standard library, JSON, unittest, Make, GitHub Actions. + +## Global Constraints + +- Current frozen source target is `08b95263c9ed700c43aea0b285696956cc23e878`. +- Current canonical G2 digest is `03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14`. +- Historical GitHub comments remain untouched; current comments supersede them append-only. +- Independent specification implementations remain irrevocable, worldwide, royalty-free, commercial and non-commercial, with mandatory LLM Errata, Thomas Willner, and repository attribution. +- Reference code under `prototype/`, `scripts/`, and `tests/` remains personal-use. +- Invitation, publication, maintainer work, and CI remain recruitment/internal evidence, never independent readiness evidence. +- G2 through G6 stay `BLOCKED` unless their existing external-evidence contracts are met. + +--- + +### Task 1: Active-Surface Manifest and Checker + +**Files:** +- Create: `publication/active-surfaces.json` +- Create: `scripts/check_publication.py` +- Create: `tests/test_publication.py` + +**Interfaces:** +- Consumes: fixed commit/digest and public URLs already verified in `docs/PUBLICATION_LOG.md`. +- Produces: `validate_manifest(payload: object) -> list[str]`, `load_manifest(path: Path) -> object`, and CLI exits `0` pass, `1` invalid, `2` inconclusive. + +- [ ] **Step 1: Write failing baseline and mutation tests** + +Create tests that require the unmodified manifest/checker to pass and mutations to reject: + +```python +class PublicationGuardRejectsDrift(unittest.TestCase): + def test_stale_commit_is_rejected(self) -> None: ... + def test_stale_digest_is_rejected(self) -> None: ... + def test_obsolete_permission_language_is_rejected(self) -> None: ... + def test_missing_required_surface_is_rejected(self) -> None: ... + def test_duplicate_mention_is_rejected(self) -> None: ... + def test_invitation_cannot_be_called_independent_evidence(self) -> None: ... +``` + +Each mutation runs `check_publication.py` against a disposable repository copy and asserts exit `1` plus a specific diagnostic label. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: + +```bash +python3 -m unittest tests.test_publication -v +``` + +Expected: import/file failures because manifest and checker do not exist. + +- [ ] **Step 3: Create exact manifest** + +Write schema version `1`, exact target/digest, licence object, and these active IDs: + +```text +g2-g3-targeted-review +g4-targeted-implementation +g5-targeted-systems +pr8-correction +discussion9-correction +``` + +Bind their exact published URLs, gates, distinct evidence roles, exact mentions, superseded URLs, and `recruitment-only` or `publication-only` boundary. + +- [ ] **Step 4: Implement minimal dependency-free checker** + +Implement strict type/key/regex/date/URL validation, exact canonical target, exact required IDs/URLs, unique roles/mentions, licence contract, and forbidden stale/permission/evidence language. Missing/unreadable JSON returns exit `2`; well-formed invalid contract returns exit `1`. + +- [ ] **Step 5: Run focused tests and verify GREEN** + +Run: + +```bash +python3 -m unittest tests.test_publication -v +python3 scripts/check_publication.py +``` + +Expected: all focused tests pass and CLI reports every publication contract check passed. + +- [ ] **Step 6: Commit manifest and checker** + +```bash +git add publication/active-surfaces.json scripts/check_publication.py tests/test_publication.py +git commit -m "feat: guard active publication surfaces" +``` + +### Task 2: Repository and CI Integration + +**Files:** +- Modify: `Makefile` +- Modify: `.github/workflows/validate.yml` +- Modify: `scripts/validate_repo.py` +- Modify: `tests/test_validate_repo.py` +- Modify: `AGENTS.md` +- Modify: `CONTRIBUTING.md` +- Modify: `docs/PUBLICATION_STRATEGY.md` + +**Interfaces:** +- Consumes: `scripts/check_publication.py` from Task 1. +- Produces: `make publication`; `make check` and CI enforce the manifest; contributor instructions preserve active/historical distinction. + +- [ ] **Step 1: Write failing integration tests** + +Add required-file mutations for the manifest, checker, and focused tests. Add focused assertions that `Makefile` routes `check` through `publication` and the workflow runs `make publication`. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: + +```bash +python3 -m unittest tests.test_validate_repo tests.test_publication -v +``` + +Expected: integration assertions fail because Make/CI wiring and required-file declarations are absent. + +- [ ] **Step 3: Wire Make, CI, and required files** + +Add: + +```make +publication: + $(PYTHON) scripts/check_publication.py +``` + +Place `publication` in `check` before `test`, add an explicit GitHub Actions step, and add all three Task 1 files to `REQUIRED_FILES`. + +- [ ] **Step 4: Document publication discipline** + +Update agent/contributor/publication documents to require: + +1. active manifest update for every supersession; +2. `git cat-file -e :` before pinned blob publication; +3. live read-after-write verification; +4. recruitment evidence never upgrades readiness. + +- [ ] **Step 5: Run integration tests and verify GREEN** + +Run: + +```bash +python3 -m unittest tests.test_validate_repo tests.test_publication -v +make publication +``` + +Expected: all focused checks pass. + +- [ ] **Step 6: Commit integration** + +```bash +git add Makefile .github/workflows/validate.yml scripts/validate_repo.py tests/test_validate_repo.py AGENTS.md CONTRIBUTING.md docs/PUBLICATION_STRATEGY.md +git commit -m "ci: enforce current publication contract" +``` + +### Task 3: Full Evidence Closure and Publication + +**Files:** +- Modify if needed: `docs/PUBLICATION_LOG.md` +- Modify ignored: `.scratch/g2-publication/loop-state.yaml` + +**Interfaces:** +- Consumes: Tasks 1 and 2. +- Produces: pushed branch, exact-head CI evidence, clean worktree, unchanged readiness verdict. + +- [ ] **Step 1: Run complete local validation** + +```bash +make check +make links +make cli-demo +git diff --check +``` + +Expected: 262 or more tests pass, all publication checks pass, all URLs resolve under repository policy, CLI repair exits expected `2`, and no whitespace error exists. + +- [ ] **Step 2: Reconcile live GitHub state** + +Verify the five active URLs render the admitted owner-authored comments, six mentions remain role-specific, and no external response arrived without analysis. + +- [ ] **Step 3: Update durable loop state** + +Record manifest/checker evidence, local validation, external response inventory, budget use, and `BLOCKED` verdict for missing external evidence. + +- [ ] **Step 4: Push branch** + +```bash +git push origin agent/g2-publication +``` + +- [ ] **Step 5: Verify exact-head CI and SHA equality** + +Require Python 3.11 and 3.13 success for exact final head, local/remote SHA equality, clean tracked worktree, and draft PR #8 pointing to exact head. + +- [ ] **Step 6: Stop without self-reference loop** + +Do not create another tracked publication-log commit solely to record the final commit or CI run. Report those externally and preserve the worktree for responses. From c79bd4e158eff36b8ce082b4805e9de4da06d492 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 02:41:27 +0200 Subject: [PATCH 53/88] feat: guard active publication surfaces --- publication/active-surfaces.json | 104 ++++++++++ scripts/check_publication.py | 320 +++++++++++++++++++++++++++++++ tests/test_publication.py | 159 +++++++++++++++ 3 files changed, 583 insertions(+) create mode 100644 publication/active-surfaces.json create mode 100644 scripts/check_publication.py create mode 100644 tests/test_publication.py diff --git a/publication/active-surfaces.json b/publication/active-surfaces.json new file mode 100644 index 0000000..04bf5db --- /dev/null +++ b/publication/active-surfaces.json @@ -0,0 +1,104 @@ +{ + "schema_version": 1, + "review_target": { + "commit": "08b95263c9ed700c43aea0b285696956cc23e878", + "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14" + }, + "license": { + "specification_implementation": "irrevocable worldwide royalty-free commercial and non-commercial", + "required_attribution": [ + "LLM Errata", + "Thomas Willner", + "https://github.com/thomaswillner/llm-errata" + ], + "reference_code": "personal-use only: prototype/, scripts/, tests/", + "case_by_case_permission_required": false + }, + "surfaces": [ + { + "id": "g2-g3-targeted-review", + "kind": "issue-comment", + "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5274367774", + "published": "2026-08-13", + "commit": "08b95263c9ed700c43aea0b285696956cc23e878", + "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", + "gates": ["G2", "G3"], + "roles": ["protocol-reviewer", "cryptography-reviewer-or-referrer"], + "mentions": ["santhoshravindran7", "jedisct1"], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269378077", + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269524323", + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269850937", + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5272345570" + ], + "evidence_boundary": "recruitment-only" + }, + { + "id": "g4-targeted-implementation", + "kind": "issue-comment", + "url": "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5274369945", + "published": "2026-08-13", + "commit": "08b95263c9ed700c43aea0b285696956cc23e878", + "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", + "gates": ["G4"], + "roles": ["adapter-author", "validator-author"], + "mentions": ["joshuaswarren", "alphaonedev"], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246", + "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269523953", + "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269524142", + "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5272475120" + ], + "evidence_boundary": "recruitment-only" + }, + { + "id": "g5-targeted-systems", + "kind": "issue-comment", + "url": "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5274371747", + "published": "2026-08-13", + "commit": "08b95263c9ed700c43aea0b285696956cc23e878", + "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", + "gates": ["G5"], + "roles": ["mem0-system-operator", "cognee-system-operator"], + "mentions": ["kartik-mem0", "Vasilije1990"], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440", + "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990", + "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156", + "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5272475614" + ], + "evidence_boundary": "recruitment-only" + }, + { + "id": "pr8-correction", + "kind": "pull-request-comment", + "url": "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5274373337", + "published": "2026-08-13", + "commit": "08b95263c9ed700c43aea0b285696956cc23e878", + "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", + "gates": ["G2", "G3", "G4", "G5", "G6"], + "roles": [], + "mentions": [], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272345782", + "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272526975" + ], + "evidence_boundary": "publication-only" + }, + { + "id": "discussion9-correction", + "kind": "discussion-comment", + "url": "https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17995307", + "published": "2026-08-13", + "commit": "08b95263c9ed700c43aea0b285696956cc23e878", + "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", + "gates": ["G2", "G3", "G4", "G5", "G6"], + "roles": [], + "mentions": [], + "supersedes": [ + "https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648" + ], + "evidence_boundary": "publication-only" + } + ] +} diff --git a/scripts/check_publication.py b/scripts/check_publication.py new file mode 100644 index 0000000..87acc9e --- /dev/null +++ b/scripts/check_publication.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""Validate the tracked active-publication-surface manifest offline.""" + +from __future__ import annotations + +import json +import re +import sys +from datetime import date +from pathlib import Path +from urllib.parse import urlparse + + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST = ROOT / "publication" / "active-surfaces.json" + +CANONICAL_COMMIT = "08b95263c9ed700c43aea0b285696956cc23e878" +CANONICAL_DIGEST = ( + "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14" +) +REPOSITORY_URL = "https://github.com/thomaswillner/llm-errata" +ALLOWED_GATES = {"G2", "G3", "G4", "G5", "G6"} +REQUIRED_ATTRIBUTION = {"LLM Errata", "Thomas Willner", REPOSITORY_URL} +REQUIRED_SURFACES = { + "g2-g3-targeted-review": { + "kind": "issue-comment", + "url": f"{REPOSITORY_URL}/issues/4#issuecomment-5274367774", + "gates": ["G2", "G3"], + "roles": ["protocol-reviewer", "cryptography-reviewer-or-referrer"], + "mentions": ["santhoshravindran7", "jedisct1"], + "evidence_boundary": "recruitment-only", + }, + "g4-targeted-implementation": { + "kind": "issue-comment", + "url": f"{REPOSITORY_URL}/issues/5#issuecomment-5274369945", + "gates": ["G4"], + "roles": ["adapter-author", "validator-author"], + "mentions": ["joshuaswarren", "alphaonedev"], + "evidence_boundary": "recruitment-only", + }, + "g5-targeted-systems": { + "kind": "issue-comment", + "url": f"{REPOSITORY_URL}/issues/6#issuecomment-5274371747", + "gates": ["G5"], + "roles": ["mem0-system-operator", "cognee-system-operator"], + "mentions": ["kartik-mem0", "Vasilije1990"], + "evidence_boundary": "recruitment-only", + }, + "pr8-correction": { + "kind": "pull-request-comment", + "url": f"{REPOSITORY_URL}/pull/8#issuecomment-5274373337", + "gates": ["G2", "G3", "G4", "G5", "G6"], + "roles": [], + "mentions": [], + "evidence_boundary": "publication-only", + }, + "discussion9-correction": { + "kind": "discussion-comment", + "url": f"{REPOSITORY_URL}/discussions/9#discussioncomment-17995307", + "gates": ["G2", "G3", "G4", "G5", "G6"], + "roles": [], + "mentions": [], + "evidence_boundary": "publication-only", + }, +} + +ROOT_KEYS = {"schema_version", "review_target", "license", "surfaces"} +TARGET_KEYS = {"commit", "surface_digest"} +LICENSE_KEYS = { + "specification_implementation", + "required_attribution", + "reference_code", + "case_by_case_permission_required", +} +SURFACE_KEYS = { + "id", + "kind", + "url", + "published", + "commit", + "surface_digest", + "gates", + "roles", + "mentions", + "supersedes", + "evidence_boundary", +} + + +def load_manifest(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def validate_manifest(payload: object) -> list[str]: + failures: list[str] = [] + if not isinstance(payload, dict) or set(payload) != ROOT_KEYS: + return ["manifest schema: root must contain the exact required fields"] + + if payload.get("schema_version") != 1: + failures.append("manifest schema: schema_version must be 1") + + target = payload.get("review_target") + target_valid = ( + isinstance(target, dict) + and set(target) == TARGET_KEYS + and target.get("commit") == CANONICAL_COMMIT + and target.get("surface_digest") == CANONICAL_DIGEST + ) + if not target_valid: + failures.append( + "review target: commit and digest must equal the corrected canonical target" + ) + + license_data = payload.get("license") + if not isinstance(license_data, dict) or set(license_data) != LICENSE_KEYS: + failures.append("licence posture: exact licence fields are required") + else: + posture = license_data.get("specification_implementation") + posture_terms = { + "irrevocable", + "worldwide", + "royalty-free", + "commercial", + "non-commercial", + } + posture_valid = ( + isinstance(posture, str) + and all(term in posture.casefold() for term in posture_terms) + and license_data.get("case_by_case_permission_required") is False + and "written" not in posture.casefold() + and "permission" not in posture.casefold() + ) + if not posture_valid: + failures.append( + "licence posture: attributed independent implementation grant must not require case-by-case permission" + ) + + attribution = license_data.get("required_attribution") + if ( + not isinstance(attribution, list) + or any(not isinstance(item, str) for item in attribution) + or set(attribution) != REQUIRED_ATTRIBUTION + or len(attribution) != len(REQUIRED_ATTRIBUTION) + ): + failures.append( + "licence attribution: LLM Errata, Thomas Willner, and repository URL are required exactly once" + ) + + reference_code = license_data.get("reference_code") + if not ( + isinstance(reference_code, str) + and "personal-use" in reference_code.casefold() + and all( + path in reference_code + for path in ("prototype/", "scripts/", "tests/") + ) + ): + failures.append( + "reference code boundary: personal-use prototype/, scripts/, and tests/ scope is required" + ) + + surfaces = payload.get("surfaces") + if not isinstance(surfaces, list): + failures.append("required active surfaces: surfaces must be a list") + return failures + + surface_ids: list[str] = [] + surface_urls: list[str] = [] + roles: list[str] = [] + mentions: list[str] = [] + for index, surface in enumerate(surfaces): + label = f"surface fields: entry {index + 1}" + if not isinstance(surface, dict) or set(surface) != SURFACE_KEYS: + failures.append(f"{label} must contain the exact required fields") + continue + + surface_id = surface.get("id") + kind = surface.get("kind") + url = surface.get("url") + published = surface.get("published") + gates = surface.get("gates") + entry_roles = surface.get("roles") + entry_mentions = surface.get("mentions") + supersedes = surface.get("supersedes") + boundary = surface.get("evidence_boundary") + + expected = REQUIRED_SURFACES.get(surface_id) if isinstance(surface_id, str) else None + valid_date = False + if isinstance(published, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", published): + try: + valid_date = date.fromisoformat(published) <= date.today() + except ValueError: + valid_date = False + valid_url = isinstance(url, str) and _is_repository_url(url) + valid_string_list_fields = all( + isinstance(value, list) + and all(isinstance(item, str) and item for item in value) + for value in (gates, entry_roles, entry_mentions, supersedes) + ) + valid_gates = ( + isinstance(gates, list) + and bool(gates) + and len(gates) == len(set(gates)) + and set(gates).issubset(ALLOWED_GATES) + ) + valid_supersedes = ( + isinstance(supersedes, list) + and bool(supersedes) + and len(supersedes) == len(set(supersedes)) + and all(_is_repository_url(item) for item in supersedes) + ) + expected_fields_match = expected is not None and all( + surface.get(field) == value for field, value in expected.items() + ) + if not ( + isinstance(surface_id, str) + and re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", surface_id) + and kind in {"issue-comment", "pull-request-comment", "discussion-comment"} + and valid_url + and valid_date + and valid_string_list_fields + and valid_gates + and valid_supersedes + and boundary in {"recruitment-only", "publication-only"} + and expected_fields_match + ): + failures.append( + f"{label} has invalid or non-canonical ID, kind, URL, date, gates, roles, mentions, supersession, or boundary" + ) + + if ( + surface.get("commit") != CANONICAL_COMMIT + or surface.get("surface_digest") != CANONICAL_DIGEST + ): + failures.append( + f"surface target binding: {surface_id!r} must bind canonical commit and digest" + ) + + if boundary == "recruitment-only" and not entry_roles: + failures.append( + f"evidence boundary: recruitment surface {surface_id!r} requires roles" + ) + if boundary == "publication-only" and (entry_roles or entry_mentions): + failures.append( + f"evidence boundary: publication surface {surface_id!r} cannot recruit or mention users" + ) + if boundary not in {"recruitment-only", "publication-only"}: + failures.append( + f"evidence boundary: {surface_id!r} cannot represent invitation or publication as independent evidence" + ) + + if isinstance(surface_id, str): + surface_ids.append(surface_id) + if isinstance(url, str): + surface_urls.append(url) + if isinstance(entry_roles, list): + roles.extend(item for item in entry_roles if isinstance(item, str)) + if isinstance(entry_mentions, list): + mentions.extend(item.casefold() for item in entry_mentions if isinstance(item, str)) + + if set(surface_ids) != set(REQUIRED_SURFACES) or len(surface_ids) != len(REQUIRED_SURFACES): + failures.append( + "required active surfaces: manifest must contain each canonical active surface exactly once" + ) + if len(surface_urls) != len(set(surface_urls)): + failures.append("unique surface URLs: active surface URLs must not repeat") + if len(roles) != len(set(roles)): + failures.append("unique evidence roles: each evidence role needs one owner") + if len(mentions) != len(set(mentions)): + failures.append("unique GitHub mentions: each identity may be notified only once") + + active_url_set = set(surface_urls) + for surface in surfaces: + if isinstance(surface, dict) and isinstance(surface.get("supersedes"), list): + if active_url_set.intersection(surface["supersedes"]): + failures.append( + "surface fields: active surface URL cannot also be superseded" + ) + break + + return failures + + +def _is_repository_url(value: object) -> bool: + if not isinstance(value, str): + return False + parsed = urlparse(value) + return ( + parsed.scheme == "https" + and parsed.netloc == "github.com" + and parsed.path.startswith("/thomaswillner/llm-errata/") + and not parsed.username + and not parsed.password + ) + + +def main() -> int: + try: + payload = load_manifest(MANIFEST) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + print(f"[INCONCLUSIVE] active publication manifest: {exc}") + return 2 + + failures = validate_manifest(payload) + if failures: + for failure in failures: + print(f"[FAIL] {failure}") + print(f"\nPublication validation failed: {len(failures)} issue(s).") + return 1 + + print("[PASS] manifest schema: exact versioned fields") + print("[PASS] review target: corrected immutable commit and digest") + print("[PASS] licence posture: implementation rights and attribution aligned") + print("[PASS] active surfaces: required URLs, roles, mentions, and boundaries") + print("\nPublication validation passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_publication.py b/tests/test_publication.py new file mode 100644 index 0000000..b79620f --- /dev/null +++ b/tests/test_publication.py @@ -0,0 +1,159 @@ +"""Negative tests for the active-publication-surface contract.""" + +from __future__ import annotations + +import json +import unittest +from collections.abc import Callable +from pathlib import Path + +from tests.support import EXIT_FAIL, EXIT_INCONCLUSIVE, EXIT_OK, check_after, repo_copy, run_checker + + +SCRIPT = "check_publication.py" + + +class PublicationGuardPasses(unittest.TestCase): + def test_unmodified_publication_manifest_passes(self) -> None: + with repo_copy() as root: + result = run_checker(root, SCRIPT) + self.assertEqual(result.returncode, EXIT_OK, result.stdout + result.stderr) + + +class PublicationGuardRejectsDrift(unittest.TestCase): + def _assert_manifest_mutation_is_rejected( + self, mutate_payload: Callable[[dict[str, object]], None], diagnostic: str + ) -> None: + def mutate(root: Path) -> None: + path = root / "publication" / "active-surfaces.json" + payload = json.loads(path.read_text(encoding="utf-8")) + mutate_payload(payload) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout + result.stderr) + self.assertIn(diagnostic, result.stdout) + + def test_stale_commit_is_rejected(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["review_target"].__setitem__( + "commit", "50e895fbfec544b16c94caa07bf2d1f4049a42e2" + ), + "review target", + ) + + def test_stale_digest_is_rejected(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["review_target"].__setitem__( + "surface_digest", + "9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804", + ), + "review target", + ) + + def test_surface_must_bind_canonical_commit_and_digest(self) -> None: + def mutate(payload: dict[str, object]) -> None: + payload["surfaces"][0]["commit"] = "a" * 40 + payload["surfaces"][0]["surface_digest"] = "b" * 64 + + self._assert_manifest_mutation_is_rejected(mutate, "surface target binding") + + def test_obsolete_permission_language_is_rejected(self) -> None: + def mutate(payload: dict[str, object]) -> None: + payload["license"]["case_by_case_permission_required"] = True + payload["license"]["specification_implementation"] = ( + "written scoped permission grant required" + ) + + self._assert_manifest_mutation_is_rejected(mutate, "licence posture") + + def test_required_attribution_is_rejected_when_incomplete(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["license"]["required_attribution"].remove( + "Thomas Willner" + ), + "licence attribution", + ) + + def test_reference_code_boundary_is_rejected_when_missing(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["license"].__setitem__( + "reference_code", "commercial reuse permitted" + ), + "reference code boundary", + ) + + def test_missing_required_surface_is_rejected(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["surfaces"].pop(), "required active surfaces" + ) + + def test_duplicate_surface_url_is_rejected(self) -> None: + def mutate(payload: dict[str, object]) -> None: + payload["surfaces"][1]["url"] = payload["surfaces"][0]["url"] + + self._assert_manifest_mutation_is_rejected(mutate, "unique surface URLs") + + def test_duplicate_evidence_role_is_rejected(self) -> None: + def mutate(payload: dict[str, object]) -> None: + payload["surfaces"][1]["roles"][0] = payload["surfaces"][0]["roles"][0] + + self._assert_manifest_mutation_is_rejected(mutate, "unique evidence roles") + + def test_duplicate_mention_is_rejected(self) -> None: + def mutate(payload: dict[str, object]) -> None: + payload["surfaces"][1]["mentions"][0] = payload["surfaces"][0]["mentions"][0] + + self._assert_manifest_mutation_is_rejected(mutate, "unique GitHub mentions") + + def test_invitation_cannot_be_called_independent_evidence(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["surfaces"][0].__setitem__( + "evidence_boundary", "independent-evidence" + ), + "evidence boundary", + ) + + def test_invalid_gate_is_rejected(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["surfaces"][0]["gates"].append("G7"), + "surface fields", + ) + + def test_invalid_publication_date_is_rejected(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["surfaces"][0].__setitem__( + "published", "13 August 2026" + ), + "surface fields", + ) + + def test_invalid_url_is_rejected(self) -> None: + self._assert_manifest_mutation_is_rejected( + lambda payload: payload["surfaces"][0].__setitem__( + "url", "https://example.invalid/review" + ), + "surface fields", + ) + + def test_missing_manifest_is_inconclusive(self) -> None: + def mutate(root: Path) -> None: + (root / "publication" / "active-surfaces.json").unlink() + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE, result.stdout) + self.assertIn("INCONCLUSIVE", result.stdout) + + def test_malformed_manifest_is_inconclusive(self) -> None: + def mutate(root: Path) -> None: + (root / "publication" / "active-surfaces.json").write_text( + "{not-json}\n", encoding="utf-8" + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE, result.stdout) + self.assertIn("INCONCLUSIVE", result.stdout) + + +if __name__ == "__main__": + unittest.main() From 3491538575730e1d8d5acb5c0bb7da32d51dfb13 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 02:44:53 +0200 Subject: [PATCH 54/88] ci: enforce current publication contract --- .github/workflows/validate.yml | 3 ++ AGENTS.md | 17 ++++++++- CONTRIBUTING.md | 17 ++++++++- Makefile | 7 +++- docs/PUBLICATION_STRATEGY.md | 17 +++++++++ scripts/validate_repo.py | 33 +++++++++++++++++ tests/test_validate_repo.py | 67 ++++++++++++++++++++++++++++++++++ 7 files changed, 157 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 6beaec3..d91fc9e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -41,5 +41,8 @@ jobs: - name: Validate production-readiness evidence run: make readiness + - name: Validate active publication surfaces + run: make publication + - name: Run self-tests, including the negative cases run: make test diff --git a/AGENTS.md b/AGENTS.md index 2462176..8364e89 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,6 +98,20 @@ Never convert missing lineage, inaccessible stores, skipped tests, timeouts, or - Respect quotation, copyright, license, confidentiality, and personal-data boundaries. - Preserve contrary and partial-collision evidence. Do not select sources only because they support the proposal. +## Publication discipline + +- `publication/active-surfaces.json` is the tracked authority for current public + review, implementation, validator, and operated-system calls. Historical + GitHub comments stay visible and are superseded append-only; never edit or + delete them to make an old target appear current. +- Before publishing a pinned GitHub blob URL, run + `git cat-file -e :` and verify the public URL. After publishing, + read the rendered artifact back and record its author, URL, body, and mentions. +- Recruitment evidence is not independent readiness evidence. Invitations, + mentions, publication, maintainer work, and CI cannot upgrade G2 through G6. +- Run `make publication` whenever an active surface, frozen target, evidence + role, mention, supersession URL, or implementation-licence posture changes. + ## Implementation and test expectations - Use synthetic fixtures; never add real personal, customer, employer, credential, or confidential data. @@ -122,11 +136,12 @@ Before presenting work as complete: make check ``` - This runs four validation components, and each answers a different question: + This runs five validation components, and each answers a different question: - `scripts/validate_repo.py` — are repository structure and metadata well formed? - `scripts/claim_guard.py` — does the documentation still state the bounded claim? It anchors to exact sentences in named files, because keyword presence over the whole corpus cannot tell a hedge from its inversion. - `scripts/check_readiness.py` — is readiness evidence structurally honest and synchronized with `PRODUCTION_READINESS.md`? + - `scripts/check_publication.py` — do active public calls bind the corrected immutable target, current licence posture, distinct evidence roles, and honest evidence boundaries? - `tests/` — do the checkers actually reject the faults they claim to catch? A checker that has only ever passed has not been shown to work. Readiness-check exit `0` validates structural honesty of the recorded evidence. It does **not** mean `PROD_READY`; the ledger and human matrix state the current verdict. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dce322c..a28c5c0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,22 @@ Never submit secrets, personal data, proprietary customer material, or confident ## Issues and pull requests -Run `make check` before opening a pull request; CI covers the same four validation components: repository structure and metadata, the bounded claim, readiness-evidence honesty, and checker self-tests. Readiness-check exit `0` validates structural honesty of the recorded evidence; it does **not** mean `PROD_READY`. If a citation was added or changed, run `make links` too, and record the pin and access date in [SOURCES.md](SOURCES.md). +Run `make check` before opening a pull request; CI covers repository structure +and metadata, the bounded claim, readiness-evidence honesty, active publication +surfaces, and checker self-tests. Run `make publication` directly when changing +an active public call. Readiness-check exit `0` validates structural honesty of +the recorded evidence; it does **not** mean `PROD_READY`. If a citation was added +or changed, run `make links` too, and record the pin and access date in +[SOURCES.md](SOURCES.md). + +Public corrections use append-only supersession: retain the historical GitHub +artifact, publish a replacement that names it as historical, and update +`publication/active-surfaces.json`. Before posting a pinned blob URL, prove the +path exists at the exact commit with `git cat-file -e :`. After +posting, perform read-after-write verification of rendered body, author, URL, +and mentions. A call, mention, or acceptance message is recruitment evidence, +not independent review, implementation, validator, system, or operational +evidence. If a checker is wrong, fix it in its own pull request with a test that fails before the fix. Widening an anchor, deleting a test, or adding a quotation exception so that an unrelated change passes is not an acceptable fix. diff --git a/Makefile b/Makefile index 3123873..1c9635a 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ PYTHON ?= python3 .DEFAULT_GOAL := check -.PHONY: check lint claim readiness test demo cli-demo links all help +.PHONY: check lint claim readiness publication test demo cli-demo links all help help: ## Show the available targets @grep -hE '^[a-z-]+:.*?## ' $(MAKEFILE_LIST) \ | awk 'BEGIN {FS = ":.*?## "}; {printf " %-8s %s\n", $$1, $$2}' -check: lint claim readiness test demo ## Everything that must pass before a change is complete +check: lint claim readiness publication test demo ## Everything that must pass before a change is complete lint: ## Structure, encoding, Markdown, links, licence, release metadata $(PYTHON) scripts/validate_repo.py @@ -29,6 +29,9 @@ claim: ## Anchored guard on the bounded novelty claim and its invariants readiness: ## Validate production-readiness evidence without upgrading the verdict $(PYTHON) scripts/check_readiness.py +publication: ## Validate active public calls without upgrading readiness + $(PYTHON) scripts/check_publication.py + test: ## Self-tests, including the negative cases each checker must reject $(PYTHON) -m unittest discover -s tests -t . diff --git a/docs/PUBLICATION_STRATEGY.md b/docs/PUBLICATION_STRATEGY.md index 47a73e7..3a0fbff 100644 --- a/docs/PUBLICATION_STRATEGY.md +++ b/docs/PUBLICATION_STRATEGY.md @@ -9,6 +9,20 @@ https://github.com/thomaswillner/llm-errata Substantive prior-art, conformance, implementation, or security findings return to traceable GitHub issues, pull requests, or private security reports. +## Active and historical surfaces + +An **Active surface** is a current public instruction recorded in +`publication/active-surfaces.json`; it binds one immutable commit and digest to +named evidence roles and an explicit evidence boundary. A **Historical surface** +records an earlier project state and remains visible for provenance, but must not +be used as current instructions. + +Corrections are append-only. Publish a superseding artifact, name the older URL +as historical, update the manifest, verify every pinned path with +`git cat-file -e :`, and read the rendered result back before +recording success. Network observations remain publication-time evidence; +`make publication` validates the deterministic tracked contract offline. + ## Message **Problem:** portable AI memory can be corrected at its source while stale local @@ -69,3 +83,6 @@ specialists to review the exact four-part conjunction and identify collisions. For every successful post, record channel, URL, publication timestamp, account, source commit, exact copy digest, and moderation state. A submitted or queued post is not recorded as publicly available until its URL is accessible. +Recruitment evidence records a bounded request only. It never establishes an +independent review, implementation, validator result, operated-system result, or +production-readiness pass. diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index de56ddc..cb699a2 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -46,6 +46,7 @@ "INDEPENDENT_IMPLEMENTATION.md", "PHASE3_SYSTEMS.md", "docs/PUBLICATION_STRATEGY.md", + "publication/active-surfaces.json", "VERSION", "LICENSE", "NOTICE", @@ -56,9 +57,11 @@ "scripts/validate_repo.py", "scripts/claim_guard.py", "scripts/check_links.py", + "scripts/check_publication.py", "tests/test_readiness.py", "tests/test_validate_repo.py", "tests/test_claim_guard.py", + "tests/test_publication.py", ".github/workflows/validate.yml", "prototype/README.md", "prototype/controller.py", @@ -426,6 +429,36 @@ def check_publication_metadata(reporter: Reporter) -> None: "and SECURITY with the dual licence contract.", ) + publication_requirements = { + "AGENTS.md": ( + "publication/active-surfaces.json", + "git cat-file -e :", + "Recruitment evidence is not independent readiness evidence", + ), + "CONTRIBUTING.md": ( + "append-only supersession", + "make publication", + "read-after-write", + ), + "docs/PUBLICATION_STRATEGY.md": ( + "Active surface", + "Historical surface", + "publication/active-surfaces.json", + ), + } + publication_alignment = all( + (ROOT / name).is_file() + and all(phrase in read_utf8(ROOT / name) for phrase in phrases) + for name, phrases in publication_requirements.items() + ) + reporter.check( + "publication discipline", + publication_alignment, + "active/historical surfaces, pinned-path proof, read-after-write, and evidence boundaries are documented", + "Restore the active-surface manifest, append-only supersession, pinned " + "blob proof, read-after-write verification, and recruitment-evidence rules.", + ) + def check_g2_independent_review_gate(reporter: Reporter) -> None: """Keep internal Phase 2 work from being represented as external review.""" diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index e33cd7f..e9c9ae7 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -108,6 +108,40 @@ def test_security_no_endorsement_boundary_cannot_be_removed(self) -> None: "Licence attribution provides security certification", ) + def test_publication_discipline_cannot_be_removed(self) -> None: + required = { + "AGENTS.md": ( + "publication/active-surfaces.json", + "git cat-file -e :", + "Recruitment evidence is not independent readiness evidence", + ), + "CONTRIBUTING.md": ( + "append-only supersession", + "make publication", + "read-after-write", + ), + "docs/PUBLICATION_STRATEGY.md": ( + "Active surface", + "Historical surface", + "publication/active-surfaces.json", + ), + } + for relative_path, phrases in required.items(): + with self.subTest(relative_path=relative_path): + def mutate(root: Path, path: str = relative_path) -> None: + target = root / path + text = target.read_text(encoding="utf-8") + for phrase in required[path]: + self.assertIn(phrase, text, f"missing publication contract: {phrase}") + target.write_text( + text.replace(required[path][0], "removed publication rule", 1), + encoding="utf-8", + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("publication discipline", result.stdout) + def test_missing_required_file_is_rejected(self) -> None: def mutate(root: Path) -> None: (root / "SECURITY.md").unlink() @@ -139,6 +173,19 @@ def mutate(root: Path, path: str = relative_path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("required files", result.stdout) + def test_missing_publication_guard_artifacts_are_rejected(self) -> None: + for relative_path in ( + "publication/active-surfaces.json", + "scripts/check_publication.py", + "tests/test_publication.py", + ): + with self.subTest(relative_path=relative_path): + def mutate(root: Path, path: str = relative_path) -> None: + (root / path).unlink() + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("required files", result.stdout) def test_internal_phase_two_completion_cannot_upgrade_g2(self) -> None: def mutate(root: Path) -> None: path = root / "readiness" / "production-readiness.json" @@ -374,5 +421,25 @@ def mutate(root: Path) -> None: self.assertIn("canonical link", result.stdout) +class PublicationGuardIntegration(unittest.TestCase): + def test_make_check_runs_publication_guard(self) -> None: + makefile = (Path(__file__).resolve().parents[1] / "Makefile").read_text( + encoding="utf-8" + ) + self.assertIn("check: lint claim readiness publication test demo", makefile) + self.assertIn( + "publication: ## Validate active public calls without upgrading readiness", + makefile, + ) + self.assertIn("$(PYTHON) scripts/check_publication.py", makefile) + + def test_ci_runs_publication_guard_explicitly(self) -> None: + workflow = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "validate.yml" + ).read_text(encoding="utf-8") + self.assertIn("- name: Validate active publication surfaces", workflow) + self.assertIn("run: make publication", workflow) + + if __name__ == "__main__": unittest.main() From c3ac91252b6381ca14ae9640448f298d298cce32 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 03:02:42 +0200 Subject: [PATCH 55/88] ci: upgrade GitHub Actions to Node 24 --- .github/workflows/links.yml | 4 ++-- .github/workflows/validate.yml | 4 ++-- CHANGELOG.md | 5 +++++ SOURCES.md | 11 ++++++++++ scripts/validate_repo.py | 40 ++++++++++++++++++++++++++++++++++ tests/test_validate_repo.py | 31 ++++++++++++++++++++++++++ 6 files changed, 91 insertions(+), 4 deletions(-) diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index b31e7b8..adbd957 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -20,10 +20,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1, Node 24 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0, Node 24 with: python-version: '3.13' diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d91fc9e..f8abdd1 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -23,10 +23,10 @@ jobs: python-version: ['3.11', '3.13'] steps: - name: Check out the repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1, Node 24 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0, Node 24 with: python-version: ${{ matrix.python-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e96d4..1d1a6b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Upgraded both GitHub Actions workflows to immutable `actions/checkout` + v7.0.1 and `actions/setup-python` v7.0.0 commit pins. Both action releases use + Node 24, removing GitHub's Node 20 deprecation path without changing the + Python 3.11/3.13 validation matrix or local Node runtime. Repository lint and + negative tests now reject missing, mutable, or downgraded action references. - Granted irrevocable, worldwide, royalty-free rights for independently authored commercial and non-commercial implementations of the specification, conditioned on accessible product attribution to LLM Errata and Thomas diff --git a/SOURCES.md b/SOURCES.md index 40c890c..3e51792 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -101,6 +101,17 @@ An archived copy of the Glean report exists at exists, the correct action is to narrow the affected row, not to keep the citation. +### GitHub Actions runtime maintenance + +These official releases were read on 2026-08-13 to remove GitHub's Node 20 +deprecation annotations. They are supply-chain maintenance evidence, not +production-readiness evidence for LLM Errata. + +| Source | Pin | Verified 2026-08-13 | +|---|---|---| +| [actions/checkout v7.0.1](https://github.com/actions/checkout/releases/tag/v7.0.1) | commit `3d3c42e5aac5ba805825da76410c181273ba90b1` | Official release published 2026-07-20; `action.yml` declares `runs.using: node24`. | +| [actions/setup-python v7.0.0](https://github.com/actions/setup-python/releases/tag/v7.0.0) | commit `5fda3b95a4ea91299a34e894583c3862153e4b97` | Official release published 2026-07-20; `action.yml` declares `runs.using: node24`. | + ## Operational-readiness grounding sources These official sources were read on 2026-08-12 to ground scope selection in diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index cb699a2..9fc95ec 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -62,6 +62,7 @@ "tests/test_validate_repo.py", "tests/test_claim_guard.py", "tests/test_publication.py", + ".github/workflows/links.yml", ".github/workflows/validate.yml", "prototype/README.md", "prototype/controller.py", @@ -109,6 +110,16 @@ "W3C PROV-DM": "https://www.w3.org/TR/prov-dm/", } +GITHUB_ACTION_PINS = { + "actions/checkout": "3d3c42e5aac5ba805825da76410c181273ba90b1", + "actions/setup-python": "5fda3b95a4ea91299a34e894583c3862153e4b97", +} + +GITHUB_WORKFLOWS = ( + ".github/workflows/links.yml", + ".github/workflows/validate.yml", +) + LOCAL_PATH_PATTERNS = ( ( "sandbox URI", @@ -356,6 +367,34 @@ def check_local_links(reporter: Reporter) -> None: ) +def check_github_actions_runtime(reporter: Reporter) -> None: + """Require reviewed immutable pins for GitHub's Node 24 action runtime.""" + + violations: list[str] = [] + for workflow_name in GITHUB_WORKFLOWS: + path = ROOT / workflow_name + if not path.is_file(): + violations.append(f"{workflow_name}: missing") + continue + text = read_utf8(path) + for action, expected_sha in GITHUB_ACTION_PINS.items(): + refs = re.findall( + rf"(?m)^\s*uses:\s*{re.escape(action)}@([^\s#]+)", text + ) + if refs != [expected_sha]: + observed = ", ".join(refs) if refs else "missing" + violations.append( + f"{workflow_name}: {action} expected {expected_sha}, got {observed}" + ) + + reporter.check( + "GitHub Actions Node 24 pins", + not violations, + "validate and links workflows pin reviewed checkout v7.0.1 and setup-python v7.0.0 commits", + "Restore immutable Node 24 action pins. " + "; ".join(violations), + ) + + def check_publication_metadata(reporter: Reporter) -> None: readme_path = ROOT / "README.md" license_path = ROOT / "LICENSE" @@ -659,6 +698,7 @@ def main() -> int: check_trailing_whitespace(reporter) check_local_paths(reporter) check_local_links(reporter) + check_github_actions_runtime(reporter) check_publication_metadata(reporter) check_g2_independent_review_gate(reporter) check_document_version_alignment(reporter, repository_version) diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index e9c9ae7..8941f69 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -441,5 +441,36 @@ def test_ci_runs_publication_guard_explicitly(self) -> None: self.assertIn("run: make publication", workflow) +class GitHubActionsRuntimeGuard(unittest.TestCase): + def test_workflows_pin_node24_action_releases(self) -> None: + workflow_dir = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" + ) + workflows = "\n".join( + path.read_text(encoding="utf-8") for path in sorted(workflow_dir.glob("*.yml")) + ) + self.assertNotIn("actions/checkout@v4", workflows) + self.assertNotIn("actions/setup-python@v5", workflows) + self.assertIn("actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", workflows) + self.assertIn("actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", workflows) + + def test_validator_rejects_deprecated_action_major(self) -> None: + def mutate(root: Path) -> None: + path = root / ".github" / "workflows" / "links.yml" + text = path.read_text(encoding="utf-8") + path.write_text( + text.replace( + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "actions/checkout@v4", + 1, + ), + encoding="utf-8", + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("GitHub Actions Node 24 pins", result.stdout) + + if __name__ == "__main__": unittest.main() From a477fe4f5c86730031b6285d9505778fb8eec060 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 14:06:23 +0200 Subject: [PATCH 56/88] fix: address external conformance findings --- AGENTS.md | 4 + CHANGELOG.md | 8 ++ CONTRIBUTING.md | 4 + IDEA.md | 10 +++ INDEPENDENT_IMPLEMENTATION.md | 10 +++ PRIOR_ART.md | 30 ++++--- PRODUCTION_READINESS.md | 2 +- README.md | 8 +- REVIEW_REQUEST.md | 6 ++ ROADMAP.md | 6 +- SOURCES.md | 3 +- THREAT_MODEL.md | 16 +++- docs/CRYPTOGRAPHY_QUALIFICATION.md | 13 ++++ docs/PUBLICATION_LOG.md | 30 +++---- prototype/README.md | 9 ++- prototype/adapters.py | 47 +++++++++++ prototype/controller.py | 116 ++++++++++++++++++++++------ prototype/errata.py | 30 +++++-- prototype/sqlite_store.py | 14 ++++ scripts/check_readiness.py | 9 ++- scripts/claim_guard.py | 10 +++ spec/README.md | 6 ++ spec/vectors/protocol-manifest.json | 14 ++++ tests/test_adapters.py | 20 +++-- tests/test_claim_guard.py | 24 ++++++ tests/test_controller.py | 97 +++++++++++++++++++++++ tests/test_errata_feed.py | 8 ++ tests/test_readiness.py | 4 +- tests/test_schema.py | 29 +++++++ 29 files changed, 514 insertions(+), 73 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8364e89..91cc4da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,6 +111,10 @@ Never convert missing lineage, inaccessible stores, skipped tests, timeouts, or mentions, publication, maintainer work, and CI cannot upgrade G2 through G6. - Run `make publication` whenever an active surface, frozen target, evidence role, mention, supersession URL, or implementation-licence posture changes. +- Targeted outreach must begin with a plain-language description of the user + problem and exact bounded question. Do not send a specialist only an internal + gate label, digest, or repository jargon and expect them to infer why the + request matters. ## Implementation and test expectations diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d1a6b5..5c821a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,14 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Corrected two conflict-disclosed external findings. Importers now remember + same-view sequence conflicts across separate observe calls, while receipts + explicitly disclaim global non-equivocation across split views. Required + adapters must establish root-specific lineage completeness before an empty + enumeration can receive `verified`; unsupported empty walks become `unknown` + with a signed limitation. The earlier categorical comparison claiming LLM + Errata was stricter than Inspeximus is withdrawn and replaced with the + verified limits and remediations in both projects. - Upgraded both GitHub Actions workflows to immutable `actions/checkout` v7.0.1 and `actions/setup-python` v7.0.0 commit pins. Both action releases use Node 24, removing GitHub's Node 20 deprecation path without changing the diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a28c5c0..3949925 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,11 +46,15 @@ A conformance change should state: - the normative behavior using clear MUST, SHOULD, or MAY language; - observable inputs, outputs, and state transitions; - failure, timeout, replay, partial-coverage, and unobservable-store behavior; +- empty enumeration with and without root-specific lineage-completeness evidence; +- same-importer conflicts separately from owner split views across importers; - security and privacy consequences; - positive, negative, and preservation tests; - backward-compatibility implications. Whenever possible, include machine-readable examples and deterministic tests. A receipt format must use the canonical terminal coverage results `verified`, `partial`, `unknown`, and `failed`. `Pending` is a lifecycle state, not a successful coverage result; stores outside the required scope are omitted. +An adapter must not infer complete coverage from an empty enumeration. It needs +explicit root-specific lineage-completeness evidence or must return `unknown`. ## Implementations diff --git a/IDEA.md b/IDEA.md index 3e8e00a..75cbab0 100644 --- a/IDEA.md +++ b/IDEA.md @@ -260,6 +260,16 @@ Signature validity authenticates the importer and receipt bytes; it does not est Coverage truthfulness requires the signed stores, aggregate, and limitations to match the declared required scope without upgrading missing or opaque evidence. +An empty enumeration is not evidence of complete lineage. A required adapter +may report `verified` for an empty root scope only when it also establishes a +root-specific write-time or audited lineage authority; otherwise the signed +store result is `unknown` with a limitation. + +A receipt establishes the signed event accepted by one importer. Importer-local +sequence checks detect conflicts visible in that view, but cannot establish +global owner non-equivocation across split views without an external witnessed +or append-only log. + The broad candidates were rejected: “a personal LLM wiki,” “portable memory,” “a memory makefile,” “verified forgetting,” “a temporal contradiction ledger,” and “self-correcting memory.” A signed post-export update feed was also rejected: EngramSpec already provides cross-runtime corrections and incremental diffs, while vCon Lifecycle/SCITT and downstream-deletion frameworks cover event and acknowledgment control planes. The surviving combination is still a synthesis of known mechanisms, so its strongest defensible claim is product and conformance design—not fundamental invention. The strongest AI-memory transport collision is [EngramSpec](https://engramspec.org/). The strongest formal standards/control-plane collision is the individual [vCon Lifecycle using SCITT draft](https://datatracker.ietf.org/doc/html/draft-howe-vcon-lifecycle-01), which records cross-recipient lifecycle events and acknowledgments backed by SCITT transparency receipts. LLM Errata profiles a boundary neither source normatively specifies: a notice reached an importer; now what must happen to the importer's locally derived AI memory, and what evidence must come back? diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index a66390a..962b7ed 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -25,6 +25,11 @@ different operations; quarantine affected state before repair; run negative, positive, and preservation checks; and report `partial`, `unknown`, or `failed` without converting missing evidence into success. +An adapter must state how it establishes root-specific lineage completeness. +Returning an empty artifact list is not enough. If the implementation cannot +show that its enumeration authority was complete for the root, it must report +`unknown` and bind the limitation into its receipt. + ## Independence and evidence An implementation report must name its authors, repository and commit, supported @@ -32,6 +37,11 @@ specification version, dependencies, test commands, unsupported behavior, and licence. Shared conformance vectors are expected. Shared reference-adapter code disqualifies the implementation as independent evidence. +One producer may supply an adapter and benchmark results, but that producer +cannot also occupy the separately authored third-party validator role for its +own implementation. Commercial interest and other conflicts must be disclosed; +they do not erase technical evidence, but they control how it can satisfy G4. + No per-implementer permission is required for an independently authored commercial or non-commercial implementation of the specification. The irrevocable implementation grant requires every product or service to credit diff --git a/PRIOR_ART.md b/PRIOR_ART.md index 674bb15..a6b6359 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -33,7 +33,7 @@ No individual mechanism is claimed as new. | [EngramSpec](https://engramspec.org/) | **Yes/Partial:** live correction endpoint and pull-based incremental diffs with tombstones; no webhook/push model in v0.1 | No requirement found for quarantine and repair of importer-local summaries, vectors, graphs, or caches | No requirement found | No repair callback found | **Strongest AI-memory transport collision.** It substantially eliminates “portable corrections” as the novel idea, but not importer-local repair conformance. | | [vCon Lifecycle using SCITT](https://datatracker.ietf.org/doc/html/draft-howe-vcon-lifecycle-01) with [SCITT RFC 9943](https://datatracker.ietf.org/doc/rfc9943/) | **Yes:** lifecycle events and acknowledgments across recipients | No AI-memory descendant-repair requirement; workflow implementation is left to applications | No | **Partial:** acknowledgments and SCITT inclusion receipts, not a semantic-repair coverage receipt | **Strongest formal control-plane collision.** It proves that cross-recipient lifecycle events and transparent receipts are not new. | | [Shomei](https://shomei.ai/how-it-works/) ([API](https://shomei.ai/docs/http-api/), [governance](https://shomei.ai/docs/governance-and-receipts/)) | **Partial:** governed correction/update inside the Shomei boundary; no reviewed persistent post-export importer subscription/callback obligation | **Yes locally:** derived lineage, erasure cascades, and explicit external-delete-pending states | **Partial:** bounded governance evidence, but no reviewed mandatory negative + positive + preservation profile for each importer | **Partial:** signed, bounded governance receipts, but not the complete cross-importer callback | **Strongest governed-memory product collision.** It establishes local lineage, lifecycle, honest coverage, and receipts. | -| [Inspeximus](https://github.com/DanceNitra/inspeximus) | **Partial:** local keyed correction/supersession and erasure channel | **Yes locally, within declared lineage:** `retract_lineage` demotes a root and its recorded descendants from default recall, retains them as superseded with `needs_rederivation`, and `rederive` can rebuild against the correction | **Partial:** stale-value and preservation-oriented checks; no reviewed mandatory replacement-activation triad across independent stores | **Partial:** signed content-free erasure evidence plus an `erasure_audit` coverage ratio, but incomplete declared lineage above zero is not itself a failing result and there is no cross-importer callback | **Strongest open-source local correction collision.** It substantially implements quarantine-then-rebuild inside one store. Its documented limit remains recorded lineage and its own boundary, not prior independent importers. | +| [Inspeximus](https://github.com/DanceNitra/inspeximus) | **Partial:** local keyed correction/supersession and erasure channel | **Yes locally, within declared lineage:** `retract_lineage` demotes a root and its recorded descendants from default recall, retains them as superseded with `needs_rederivation`, and `rederive` can rebuild against the correction | **Partial:** stale-value and preservation-oriented checks; no reviewed mandatory replacement-activation triad across independent stores | **Partial:** signed content-free erasure evidence; current `erasure_audit` demotes known unresolved derivation holes but still cannot prove subject-specific completeness after a correct content-free cascade, and there is no cross-importer callback | **Strongest open-source local correction collision.** It substantially implements quarantine-then-rebuild inside one store. Its documented limit remains recorded lineage and its own boundary, not prior independent importers. | | [MemoRepair](https://arxiv.org/abs/2605.07242v1) | No cross-system delivery | **Yes, as an explicit contract:** descendants withdrawn before repair, republication restricted to validated predecessor-closed successors; invalidated-memory exposure cut from 69.8–94.3% to 0% *given complete influence provenance* | **Partial:** validated republication, no preservation test | No | **Strongest requirement-B collision found.** It independently arrives at quarantine-before-repair. Added 2026-08-07; post-dates the original cutoff. | | [Governed Evolving Memory](https://arxiv.org/abs/2605.26252v1) | No | **Partial:** formal correctness conditions for dependency consistency and provenance preservation | No | No | Argues record-level stores cannot satisfy those conditions. | | [Always-On Agents / AOEP-v0](https://arxiv.org/abs/2606.30306v1) | No | Governance obligations scored, not implemented | **Partial:** a deterministic evaluation contract scoring state mutation and recovery rather than answer quality | No | **Closest conformance-protocol collision.** | @@ -93,14 +93,26 @@ the corrected root. That is a stronger requirement-B collision than the previous comparison to `forget_subject` and should be treated as local quarantine-then-rebuild, not merely deletion. -The same pinned documentation describes `erasure_audit()` returning -`coverage{records, with_declared_lineage, undeclared_derived, declared_ratio}`. -Its tests force `unaudited` when declared lineage is zero, but a nonzero -incomplete ratio can still return `no_declared_residue`. LLM Errata's required- -store aggregation is therefore stricter: unresolved required coverage cannot -become aggregate success. The Inspeximus maintainer additionally disclosed a -0.0000 declared-lineage ratio in its dogfood store; this repository records that -as interested-party testimony, not independently reproduced code evidence. +The earlier pinned documentation described `erasure_audit()` returning +`coverage{records, with_declared_lineage, undeclared_derived, declared_ratio}` +and allowed a nonzero incomplete ratio to return `no_declared_residue`. The +maintainer accepted that finding and changed commit +[`36611027a463a8e526e23baf2d6bb8d9797b67ac`](https://github.com/DanceNitra/inspeximus/commit/36611027a463a8e526e23baf2d6bb8d9797b67ac): known unresolved derivation holes +now return `partially_audited`, and the audit reports +`subject_reachable_records`. The maintainer also found that zero subject reach +cannot gate success because a correct content-free cascade can erase the same +evidence needed to distinguish “nothing was declared” from “everything was +erased.” That residual limit is material. + +The same conflict-disclosed review found a mirror defect in LLM Errata: an +adapter could enumerate an empty set without proving root-specific lineage +completeness and still receive `verified`. The earlier sentence claiming this +repository's aggregation was categorically stricter was therefore false. The +reference controller now maps that unsupported empty walk to `unknown`, while +preserving the four public coverage results. Both projects still trust their +own adapter or store instrumentation and neither result is independent +certification. The maintainer's 0.0000 dogfood ratio remains interested-party +testimony, not independently reproduced evidence. These mechanisms apply to explicitly keyed or successfully extractor-keyed assertions and recorded lineage. The documented scope is one store, not every diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index bb6cc87..a71b2d5 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 is internally complete: schemas, CLI quarantine checkpoints, semantic probes, key rotation, concurrency, invalid-target, confidentiality, and complete receipt-field binding vectors are implemented; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation and unsupported empty enumeration, plus schemas, checkpoints, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/README.md b/README.md index b5c1162..416f6d7 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,12 @@ observe → quarantine → rebuild → test → attest An importer that cannot inspect a relevant cache or derived store reports `unknown`. It does not silently turn incomplete coverage into success. +An empty enumeration is not a clean result by itself. The adapter must also +establish that its root-specific lineage authority is complete; otherwise the +store remains `unknown`. A receipt also speaks only for the signed feed view +that one importer accepted. Detecting different views delivered to different +importers requires an external witness or append-only transparency mechanism. + ## The repair triad Every repair is evaluated against three postconditions: @@ -167,7 +173,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface are implemented. Phase 2 includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, concurrent-conflict and invalid-target cases, content-free confidentiality evidence, and mutation coverage for every signed receipt field. G2 remains `BLOCKED` pending dated independent review of the exact complete conformance surface. +Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface are implemented with two conflict-disclosed external findings remediated: split-view equivocation is now stated as outside importer-local proof, and empty enumeration cannot become verified without root-specific lineage-completeness evidence. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, and mutation coverage for every signed receipt field. G2 remains `BLOCKED`: the reviewer disclosed a commercial conflict and did not cover the full required conformance scope, so a qualifying independent review is still absent. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md index b09c303..4a81cbb 100644 --- a/REVIEW_REQUEST.md +++ b/REVIEW_REQUEST.md @@ -29,6 +29,12 @@ lineage gaps, opaque coverage, receipt binding, erasure confidentiality, key lifecycle, retries, and incomplete stores. Find the smallest counterexample that breaks an invariant or makes the contract impractical. +The current reference boundary distinguishes same-importer conflict detection +from global non-equivocation. A receipt proves which signed event one importer +accepted; it does not prove every importer received the same event. Also test +whether an empty adapter enumeration has root-specific lineage-completeness +evidence. Empty-without-evidence must be `unknown`, not `verified`. + ## Required review record Please identify: diff --git a/ROADMAP.md b/ROADMAP.md index 79e2188..c127d38 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -104,7 +104,7 @@ Kill or redesign the concept if exact lineage cannot survive even the controlled ## Phase 2 — Conformance surface -**Status:** internally complete, externally unreviewed. `spec/` carries schemas, static vectors, executable stateful vectors, and deterministic semantic fixtures. `prototype/cli.py` exposes the declared control-plane commands and `semantic-test`; `prototype/checkpoints.py` makes `errata quarantine` durable and requires its state-bound evidence before CLI repair; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. `OwnerKeySchedule` binds every event to the key active at its sequence. Stateful vectors execute valid rotation, rotated-key reuse, concurrent sequence conflict, invalid targets, content-free erasure evidence, and mutation of every signed receipt field. This internal work does not complete G2: a dated independent external conformance review of the exact complete Phase 2 surface remains required. +**Status:** implemented with external findings remediated; qualifying review still incomplete. `spec/` carries schemas, static vectors, executable stateful vectors, and deterministic semantic fixtures. `prototype/cli.py` exposes the declared control-plane commands and `semantic-test`; `prototype/checkpoints.py` makes `errata quarantine` durable and requires its state-bound evidence before CLI repair; `prototype/semantic.py` keeps model-assisted probes provider-neutral, configuration-bound, and fail-closed. `OwnerKeySchedule` binds every event to the key active at its sequence. Stateful vectors execute valid rotation, rotated-key reuse, same-view sequence conflict, invalid targets, content-free erasure evidence, and mutation of every signed receipt field. Conflict-disclosed external review found that split views were overstated and an empty enumeration could receive `verified` without lineage-completeness evidence; both findings now have fail-closed tests and corrected claims. This work does not complete G2: the reviewer disclosed a commercial conflict and did not cover the full required scope, so a dated qualifying independent review remains required. Only after the file-backed proof passes: @@ -124,7 +124,7 @@ Only after the file-backed proof passes: errata audit ``` -4. Define an adapter interface for enumeration, quarantine, reconstruction, verification, and coverage reporting. +4. Define an adapter interface for enumeration, root-specific lineage-completeness evidence, quarantine, reconstruction, verification, and coverage reporting. Empty enumeration without that evidence is `unknown`. 5. Publish conformance vectors for signatures, sequencing, key rotation, concurrent events, invalid targets, receipt binding, and confidentiality. 6. Add model-assisted semantic probes behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. **Implemented internally:** [`prototype/semantic.py`](prototype/semantic.py), [`spec/semantic/`](spec/semantic/), and `errata semantic-test` record only structured, configuration-bound observations; inconclusive, malformed, missing, duplicate, or drifted required evidence is not success. @@ -192,7 +192,7 @@ Every phase must account for: - issuer authorization and delegated authority; - key rotation, recovery, and compromise; -- feed rollback and equivocation; +- feed rollback, same-view conflicts, and split-view equivocation requiring an external witness or transparency mechanism; - poisoned or instruction-bearing payloads; - denial of service through excessive repair events; - privacy leakage through identifiers, polling, registrations, and receipts; diff --git a/SOURCES.md b/SOURCES.md index 3e51792..23c7ed0 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -48,7 +48,8 @@ the default branch and will move. | Source | Pin | Verified 2026-08-07 | |---|---|---| -| [Inspeximus](https://github.com/DanceNitra/inspeximus) | `4c711f2982911841d86d7ac1989b0ffb866dc891` (observed 2026-08-12) | `docs/API.md` confirms `retract_lineage` demotion, `include_superseded`, `needs_rederivation`, and `rederive`; it also documents `erasure_audit` coverage fields and zero-lineage `unaudited`. Tests confirm zero declared lineage is refused while incomplete nonzero lineage can report `no_declared_residue`. This commit post-dates the 2026-08-01 cutoff and is supplementary collision evidence, not a backdated change to the search corpus. | +| [Inspeximus](https://github.com/DanceNitra/inspeximus) | `4c711f2982911841d86d7ac1989b0ffb866dc891` (review baseline, 2026-08-12); remediation `36611027a463a8e526e23baf2d6bb8d9797b67ac` (2026-08-13) | Baseline confirms `retract_lineage`, `include_superseded`, `needs_rederivation`, and `rederive`. Remediation source sets package version 2.6.0, makes known unresolved derivation holes `partially_audited`, reports `subject_reachable_records`, and includes negative controls explaining why a global declared-ratio threshold would be wrong. No public GitHub release/tag was found, so 2.6.0 is source-version evidence, not a verified release artifact. Both commits post-date the 2026-08-01 cutoff and are supplementary collision evidence. | +| [jedisct1/ed25519.py](https://github.com/jedisct1/ed25519.py) | `67902d339ea47418a60fb7684255b81bc4f6d46e` (2026-08-13) | Maintainer-provided pure-Python Ed25519 comparison implementation. README claims RFC 8032 vectors, canonical/small-order rejection, batch verification, and optional randomized signing countermeasures. Repository supplies no detected licence file or independent audit/build qualification, so it is a comparison oracle only and does not satisfy G3. | | [Doyle, A Truth Maintenance System](https://dblp.org/rec/journals/ai/Doyle79.html) | Artificial Intelligence 12(3), 1979; DBLP `journals/ai/Doyle79` | Stable bibliographic record confirms title and publication. Cited by Inspeximus as prior retract-and-retain lineage. Included to make historical dependency-retraction collision explicit; LLM Errata claims no invention of this mechanism. | | [IAB Data Deletion Request Framework](https://github.com/InteractiveAdvertisingBureau/Data-Subject-Rights/blob/main/Data%20Deletion%20Request%20Framework.md) | `b9418f5394ca91193181a61c567ffbbdff79cdef` (2025-02-06) | Repository unchanged since well before the cutoff | | [Karpathy LLM Wiki idea file](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) | Gist, revision not pinned | Cited only as a presentation precedent, so drift carries no claim risk | diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 9c05dc7..da5b57d 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -24,7 +24,7 @@ mitigation or an admitted limit. | Forged erratum | Ed25519 over a canonical serialisation; the feed is refused, not the entry | `errata.verify_feed` | | Tampering after signing | The signature covers every semantic field; `signature` is excluded from its own preimage | `Erratum.signable` | | Replay of a retired state | Monotonic sequence; a lower sequence is a rollback and refused | `errata.verify_feed` | -| Equivocation | Two different errata at one sequence refuse the whole feed | `errata.verify_feed` | +| Same-view conflict or splice | Different signed event content at one sequence is remembered and refused when one importer sees it, including across separate observe calls | `errata.verify_feed`, `Importer.observe` | | Suppressed erratum | A sequence gap is refused: the missing entry may be the one retiring the state about to be served | `errata.verify_feed` | | Erasure smuggling content back | An erasure carrying a replacement is refused | `errata._check_shape` | | Signature malleability | Non-canonical scalars (`S >= L`) rejected, so a receipt cannot be altered and still verify | `ed25519.verify` | @@ -34,10 +34,19 @@ mitigation or an admitted limit. | Stale export undoing a repair | A re-import at or below the applied sequence is refused | `controller.reimport` | | Resuming as an authentication bypass | `resume` re-runs `observe`; it relaxes nothing | `controller.repair` | | Silent partial coverage | A required store that is not `verified` prevents aggregate success | `receipts.aggregate_coverage` | +| Empty walk presented as complete | Enumeration without explicit root-specific lineage-completeness evidence is signed as `unknown`, even when the adapter returns zero artifacts or claims `verified` | `controller._lineage_limitation`, `controller._attest` | | Erased value surviving in evidence | Receipts carry no digest of erased content; low-entropy propositions cannot be safely committed to, so nothing is committed | `signing.commitment` | ## Not defended, and why +**Owner split-view equivocation.** Two importers can receive different, +correctly signed events at the same sequence and each can hold a locally +monotone, gap-free feed. No importer can detect an event it never received. +Every receipt therefore states that it authenticates only that importer's +accepted feed view. Establishing global owner non-equivocation requires an +external witness, gossip protocol, or append-only transparency log; none is +implemented by this repository. + **A lying importer.** An importer that signs `verified` over a repair it never performed produces a receipt that verifies. Nothing in this design detects it. The signature identifies who to disbelieve, which is the whole of what it @@ -51,7 +60,10 @@ soft-deleted embeddings remain reconstructible from HNSW index files in ChromaDB, FAISS and Weaviate after the API reports the record gone. An adapter that reports `verified` on the strength of a delete response is honest and wrong. This is the sharpest open problem in the design: **coverage honesty -currently has no adversary**, and an adapter is trusted to characterise itself. +still has no independent adversary**, and an adapter is trusted to characterise +itself. The controller now refuses to infer lineage completeness from an empty +enumeration, but a dishonest adapter can still falsely attest that its +root-specific lineage authority is complete. **Unregistered copies.** The closure is *known* because lineage is written at derivation time. Prose copied by a human, a screenshot, a backup, a provider diff --git a/docs/CRYPTOGRAPHY_QUALIFICATION.md b/docs/CRYPTOGRAPHY_QUALIFICATION.md index 800ee63..8cdd638 100644 --- a/docs/CRYPTOGRAPHY_QUALIFICATION.md +++ b/docs/CRYPTOGRAPHY_QUALIFICATION.md @@ -83,6 +83,19 @@ The assessment makes libsodium the stronger audited-lineage candidate. It does not justify describing 1.0.22 or an application binding as independently audited without reviewing changes since 1.0.13 and the exact production build. +### Maintainer-provided pure-Python comparison + +In response to the review request, libsodium maintainer Frank Denis pointed to +[`jedisct1/ed25519.py`](https://github.com/jedisct1/ed25519.py) at commit +`67902d339ea47418a60fb7684255b81bc4f6d46e`. Its documentation describes RFC +8032 vectors, canonical and small-order refusal, batch verification, and +optional randomized signing countermeasures. This is relevant design and +compatibility evidence, but it does not change the production decision: it is +still pure Python, no independent audit or exact-build side-channel review was +found, and no repository licence file was detected. It may be used as a +comparison oracle after its licensing status is clarified; it is not the G3 +production backend. + ## Required production design A qualifying implementation must preserve these contracts: diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index dcb75c8..e9846b5 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -24,6 +24,9 @@ readiness gate. Last verified: 2026-08-13. | GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | | Historical Phase 2 review target | Commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2), digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` | Stale: this target predated the accepted prior-art corrections and must not be used for a current review. Preserved as historical publication evidence. | | Corrected complete Phase 2 review target | Commit [`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878), digest `03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14` | Complete corrected specification, prior-art, licensing, and cryptographic-qualification surface. Checkout and committed-source digests match; exact-target CI passed Python 3.11 and 3.13. Internal evidence does not satisfy G2. | +| Interested-party technical review | [Two counterexamples](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5276848751) | Bound to the corrected target with explicit Inspeximus/market conflict. Reproduced same-importer versus split-view equivocation limits and unsupported empty-enumeration success. Useful external technical evidence, but not a qualifying complete independent G2 review. | +| Cross-project remediation and adapter offer | [Inspeximus follow-up](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5279081884) | Inspeximus source commit `3661102` fixes an adjacent known-hole audit defect and offers an independently authored adapter plus benchmark harness with explicit commercial interest. No adapter artifact exists yet; the producer cannot also validate its own adapter for G4. | +| Cryptography maintainer response | [Pure-Python Ed25519 reference](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5275036522) | Useful referral and evidence that the outreach lacked plain-language context. The linked pure-Python project is a comparison oracle, not audited constant-time G3 evidence. | | Final independent-implementation call | [Issue #5 completion update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5272475120) | Publishes the immutable Phase 2 commit and digest for two independent adapters and a separate validator. No implementation has been accepted; G4 remains `BLOCKED`. | | Final three-system nomination call | [Issue #6 completion update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5272475614) | Publishes the immutable Phase 2 target and exact nomination fields. No system authorization or experiment evidence exists; G5 remains `BLOCKED`. | | Interested-party review follow-up | [Inspeximus maintainer reply](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5272476094) | Acknowledges the source-grounded correction and points to the final review target. Conflict disclosure remains controlling; a future review from this maintainer cannot satisfy G2 alone. | @@ -36,20 +39,23 @@ readiness gate. Last verified: 2026-08-13. | Phase 2 completion announcement | [Discussion #9 completion update](https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648) | Announces internal completion and routes falsifiers to Issues #4, #5, #6, and #10. Publication only; it records no external acceptance or readiness-gate change. | | Six-step readiness checkpoint | [PR #8 checkpoint](https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272526975) | Audits every approved production-readiness step against exact internal and external evidence. Records `NOT_PROD_READY`, the external blockers, PR #3 status, validation, and publication boundaries without claiming certification. | -One targeted invitation has the interested-party response recorded above; it -is not an acceptance of an independent reviewer role and does not satisfy G2. +One targeted invitation now has a conflict-disclosed technical review and +implementation offer recorded above; it does not satisfy G2 or G4 by itself. The corrected follow-ups use six role-specific mentions across three issues and -no mentions in PR #8 or Discussion #9. No response to those new follow-ups, -adapter, validator, operated-system approval, or CODEOWNER volunteer is recorded. +no mentions in PR #8 or Discussion #9. No produced adapter, separate validator, +operated-system approval, or CODEOWNER volunteer is recorded. -The current complete conformance review target is source commit +The former complete conformance review target is source commit [`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878) with canonical Phase 2 surface digest `03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14`. The digest was recomputed both from the checkout and from `git show` at that commit with an exact match. The exact-target [GitHub Actions run 31646616085](https://github.com/thomaswillner/llm-errata/actions/runs/31646616085) -passed Python 3.11 and Python 3.13. The previously frozen source commit +passed Python 3.11 and Python 3.13. It is now historical because external review +found two accepted defects in that surface; a remediation target will supersede +it after the corrected source is committed, digested, tested, and published. +The previously frozen source commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2) and digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` are historical and stale because they predate the accepted prior-art @@ -106,11 +112,9 @@ interoperability evidence. ## Readiness boundary -GitHub calls, invitations, CI results, and internal reviews are not independent -external evidence. Phase 2 is internally complete: explicit quarantine -checkpoints, key rotation, concurrent conflicts, invalid targets, -confidentiality, semantic probes, and every-field receipt binding are covered. -G2 remains `BLOCKED` pending a dated independent external review of exact commit -`08b95263c9ed700c43aea0b285696956cc23e878` and digest -`03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14`. +GitHub calls, invitations, CI results, maintainer work, and conflict-disclosed +interested-party review are not qualifying independent evidence. Phase 2 +implementation now includes remediation for same-view versus split-view +equivocation and unsupported empty enumeration. G2 remains `BLOCKED` pending a +dated qualifying independent review of the exact post-remediation target. Repository verdict remains `NOT_PROD_READY`. diff --git a/prototype/README.md b/prototype/README.md index e7fce33..f123c1c 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -31,7 +31,7 @@ non-green result cannot be mistaken for a bug in the aggregation. | Module | Responsibility | |---|---| -| `errata.py` | The trust boundary. Rejects forgery, tampering, rollback, sequence gaps, equivocation, unknown targets, and operations that carry the wrong shape. | +| `errata.py` | The importer-view trust boundary. Rejects forgery, tampering, rollback, sequence gaps, conflicts visible to that importer, unknown targets, and operations that carry the wrong shape. It cannot detect a different signed view delivered only to another importer. | | `lineage.py` | Exact lineage recorded at import and derivation time, not reconstructed afterwards. Supplies the known derivation closure and the inputs that survive a retirement. | | `adapters.py` | The three stores, plus the four coverage results. | | `strategies.py` | How a repair is carried out. One conforming strategy and three that are not, because a conformance suite where nothing can fail has not tested anything. | @@ -87,6 +87,13 @@ consumed, wrong-target, state-drifted, adapter-drifted, or ungated evidence. Consumption happens only after receipt and applied-state writeback, so an interrupted rebuild retains an unconsumed checkpoint for safe resume. +Enumeration is necessary but not sufficient for verified coverage. An adapter +must also expose `lineage_complete(root) -> True`, backed by a write-time or +audited root-specific lineage authority. Missing or false evidence makes that +required store `unknown`, even when enumeration returns an empty tuple and the +adapter's own coverage method claims `verified`. This is an adapter attestation, +not independent proof against a dishonest store. + Exit codes are part of the interface. `0` is success, `1` is a refusal or a failed check, and **`2` means the repair ran and the result is not verified**. `2` is not a lesser `1`: it is the case the whole proposal exists to make diff --git a/prototype/adapters.py b/prototype/adapters.py index a2b3ab3..eff4971 100644 --- a/prototype/adapters.py +++ b/prototype/adapters.py @@ -21,6 +21,7 @@ import re from dataclasses import dataclass from enum import Enum +from typing import Protocol from prototype.lineage import LineageLedger @@ -61,6 +62,28 @@ class HistoricalHit: valid_until: str +class StoreAdapter(Protocol): + """Minimum fail-closed contract for one required retrieval store.""" + + name: str + required: bool + + def enumerate(self, root: str) -> tuple[str, ...]: ... + + def lineage_complete(self, root: str) -> bool: + """Whether enumeration is complete under a root-specific authority.""" + + ... + + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: ... + + def is_quarantined(self, artifact_id: str) -> bool: ... + + def coverage(self, root: str) -> Coverage: ... + + def dispositions(self, root: str) -> dict[str, str]: ... + + class MarkdownAdapter: """A file-backed store with exact lineage.""" @@ -83,6 +106,15 @@ def enumerate(self, root: str) -> tuple[str, ...]: ) ) + def lineage_complete(self, root: str) -> bool: + """The write-time ledger is this adapter's enumeration authority.""" + + return root in self._ledger.roots_seen() and all( + artifact.artifact_id in self._ledger.descendants(root) + for artifact in self._ledger.artifacts() + if artifact.store == self.name and artifact.root == root + ) + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: self._quarantined.update(artifact_ids) @@ -228,6 +260,18 @@ def enumerate(self, root: str) -> tuple[str, ...]: ) ) + def lineage_complete(self, root: str) -> bool: + """Every indexed entry records its source in ``_source_of`` at write time.""" + + return ( + root in self._ledger.roots_seen() + and set(self._text).issubset(self._source_of) + and all( + source in self._ledger.artifact_ids() + for source in self._source_of.values() + ) + ) + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: self._quarantined.update(artifact_ids) @@ -325,6 +369,9 @@ def acknowledge(self, root: str) -> bool: self._acknowledged.add(root) return True + def lineage_complete(self, root: str) -> bool: + return False + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: return None diff --git a/prototype/controller.py b/prototype/controller.py index 8cb9d1d..c0afe4e 100644 --- a/prototype/controller.py +++ b/prototype/controller.py @@ -17,13 +17,20 @@ from enum import Enum from typing import Any, Sequence -from prototype.adapters import CannotEnumerate +from prototype.adapters import CannotEnumerate, Coverage, StoreAdapter from prototype.checkpoints import ( AdapterCheckpoint, CheckpointError, QuarantineCheckpoint, ) -from prototype.errata import Erratum, FeedError, Operation, RootRegistry, verify_feed +from prototype.errata import ( + Erratum, + FeedError, + Operation, + RootRegistry, + event_fingerprint, + verify_feed, +) from prototype.lineage import LineageLedger from prototype.receipts import Receipt, aggregate_coverage from prototype.signing import Signer, VerificationKey, commitment @@ -53,7 +60,7 @@ def __init__( name: str, *, ledger: LineageLedger, - adapters: Sequence[Any], + adapters: Sequence[StoreAdapter], signer: Signer, owner: VerificationKey, roots: RootRegistry, @@ -69,6 +76,7 @@ def __init__( self.journal: list[JournalEvent] = [] self.last_sequence = 0 self._applied: dict[str, int] = {} + self._observed_sequences: dict[int, tuple[str, str]] = {} # -- accessors ----------------------------------------------------- @@ -114,6 +122,16 @@ def state_root(self) -> str: # -- the loop ------------------------------------------------------ def observe(self, erratum: Erratum) -> Erratum: + fingerprint = event_fingerprint(erratum) + prior = self._observed_sequences.get(erratum.sequence) + if prior is not None and prior[1] != fingerprint: + self.journal.append( + JournalEvent(Phase.REFUSED, {"erratum": erratum.erratum_id}) + ) + raise FeedError( + f"sequence {erratum.sequence} conflict in this importer view: " + f"{prior[0]!r} and {erratum.erratum_id!r} carry different signed events" + ) try: accepted = verify_feed( [erratum], @@ -126,6 +144,9 @@ def observe(self, erratum: Erratum) -> Erratum: JournalEvent(Phase.REFUSED, {"erratum": erratum.erratum_id}) ) raise + self._observed_sequences.setdefault( + erratum.sequence, (erratum.erratum_id, fingerprint) + ) self.journal.append( JournalEvent( Phase.OBSERVE, @@ -155,11 +176,11 @@ def quarantine(self, erratum: Erratum) -> QuarantineCheckpoint: validated = self.observe(erratum) pre_state_root = self.state_root() root = validated.target_root - gated, unenumerable = self._quarantine(root) - unknown = set(unenumerable) + gated, coverage_limitations = self._quarantine(root) + unknown = set(coverage_limitations) records = [] for adapter in sorted(self.adapters, key=lambda item: item.name): - limitation = self._opaque_limitation(adapter.name) if adapter.name in unknown else None + limitation = coverage_limitations.get(adapter.name) records.append( AdapterCheckpoint( name=adapter.name, @@ -190,9 +211,11 @@ def repair_quarantined( gated = { record.name: list(record.artifact_ids) for record in checkpoint.adapters } - unenumerable = [ - record.name for record in checkpoint.adapters if record.coverage == "unknown" - ] + coverage_limitations = { + record.name: record.limitation + for record in checkpoint.adapters + if record.coverage == "unknown" and record.limitation is not None + } self.journal.append(JournalEvent(Phase.REBUILD_BEGIN, {"root": root})) self.strategy.apply(self, validated, gated) @@ -202,7 +225,7 @@ def repair_quarantined( self.journal.append(JournalEvent(Phase.TEST, dict(triad))) receipt = self._attest( - validated, checkpoint.pre_state_root, unenumerable, triad + validated, checkpoint.pre_state_root, coverage_limitations, triad ) self.last_sequence = validated.sequence self._applied[root] = validated.sequence @@ -243,7 +266,12 @@ def _validate_checkpoint( ): raise CheckpointError(f"checkpoint opaque coverage drifted: {name}") continue - if record.coverage != "verified" or record.limitation is not None: + current_limitation = self._lineage_limitation(adapter, erratum.target_root) + expected_coverage = "unknown" if current_limitation else "verified" + if ( + record.coverage != expected_coverage + or record.limitation != current_limitation + ): raise CheckpointError(f"checkpoint adapter coverage drifted: {name}") if record.artifact_ids != current: raise CheckpointError(f"checkpoint gated artifact set drifted: {name}") @@ -257,12 +285,43 @@ def _opaque_limitation(name: str) -> str: "is unknown and no repair elsewhere changes that" ) - def _quarantine(self, root: str) -> tuple[dict[str, list[str]], list[str]]: + @staticmethod + def _lineage_limitation(adapter: StoreAdapter, root: str) -> str | None: + """Return a binding limitation unless root-specific lineage is audited. + + Successful enumeration is not itself evidence that the enumeration + source was complete. An adapter must explicitly bind its walk to a + write-time or otherwise audited lineage authority. The method remains + an adapter attestation, not proof against a dishonest adapter. + """ + + audit = getattr(adapter, "lineage_complete", None) + try: + complete = audit(root) is True if audit is not None else False + except Exception: + complete = False + if complete: + return None + return ( + f"{adapter.name}: enumeration returned a result but the adapter did " + f"not establish complete root-specific lineage for {root}; empty or " + "partial walks cannot become verified coverage" + ) + + @staticmethod + def _feed_view_limitation() -> str: + return ( + "receipt authenticates this importer's accepted feed view only; " + "importer-local sequencing cannot establish global owner " + "non-equivocation without an external witnessed or append-only log" + ) + + def _quarantine(self, root: str) -> tuple[dict[str, list[str]], dict[str, str]]: """Gate every known descendant everywhere, before any rebuild starts.""" self.journal.append(JournalEvent(Phase.QUARANTINE_BEGIN, {"root": root})) gated: dict[str, list[str]] = {} - unenumerable: list[str] = [] + limitations: dict[str, str] = {} for adapter in self.adapters: try: descendants = adapter.enumerate(root) @@ -270,19 +329,22 @@ def _quarantine(self, root: str) -> tuple[dict[str, list[str]], list[str]]: acknowledge = getattr(adapter, "acknowledge", None) if acknowledge is not None: acknowledge(root) - unenumerable.append(adapter.name) + limitations[adapter.name] = self._opaque_limitation(adapter.name) gated[adapter.name] = [] continue + lineage_limitation = self._lineage_limitation(adapter, root) + if lineage_limitation is not None: + limitations[adapter.name] = lineage_limitation adapter.quarantine(descendants) gated[adapter.name] = list(descendants) self.journal.append(JournalEvent(Phase.QUARANTINE_COMPLETE, gated)) - return gated, unenumerable + return gated, limitations def _attest( self, erratum: Erratum, pre_state_root: str, - unenumerable: Sequence[str], + coverage_limitations: dict[str, str], triad: dict[str, str], ) -> Receipt: root = erratum.target_root @@ -290,11 +352,19 @@ def _attest( # rather than congratulated as 'not applicable.'" Omission is the only # honest way to leave a store out; there is no `not-applicable` result, # because there is no way to distinguish it from an unchecked one. - stores = { - adapter.name: adapter.coverage(root) - for adapter in self.adapters - if getattr(adapter, "required", True) - } + stores = {} + for adapter in self.adapters: + if not getattr(adapter, "required", True): + continue + reported = adapter.coverage(root) + # Missing lineage evidence may downgrade a claimed success, never + # upgrade a failure the adapter already reported. + stores[adapter.name] = ( + Coverage.UNKNOWN + if adapter.name in coverage_limitations + and reported is Coverage.VERIFIED + else reported + ) receipt = Receipt( importer=self.name, erratum_id=erratum.erratum_id, @@ -310,8 +380,8 @@ def _attest( triad=triad, aggregate=aggregate_coverage(stores, triad), limitations=[ - self._opaque_limitation(name) - for name in unenumerable + *sorted(coverage_limitations.values()), + self._feed_view_limitation(), ], history_retained=erratum.operation is Operation.SUPERSEDE, adapter_versions={adapter.name: "0.1.0" for adapter in self.adapters}, diff --git a/prototype/errata.py b/prototype/errata.py index efe6972..8f498f2 100644 --- a/prototype/errata.py +++ b/prototype/errata.py @@ -1,7 +1,7 @@ """The errata feed: an append-only, monotonically sequenced channel of authorised corrections, supersessions, and erasures for exported memory roots. -This module is the trust boundary. Everything downstream — quarantine, rebuild, +This module is one importer-view trust boundary. Everything downstream — quarantine, rebuild, probes, receipts — assumes that an erratum which reached the controller was authorised by the owner, arrived in order, and named exactly one known root. @@ -12,6 +12,7 @@ from __future__ import annotations import json +import hashlib from dataclasses import dataclass, replace from enum import Enum from typing import Any, Iterable, Iterator, Mapping, Sequence @@ -138,6 +139,15 @@ def read_feed(text: str) -> list[Erratum]: return [Erratum.from_json(line) for line in text.splitlines() if line.strip()] +def event_fingerprint(erratum: Erratum) -> str: + """Bind one sequence to the complete signed event, not only its ID.""" + + canonical = json.dumps( + erratum.signable(), sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + def _check_shape(erratum: Erratum, roots: RootRegistry) -> None: if erratum.target_root not in roots: raise FeedError( @@ -178,14 +188,14 @@ def verify_feed( ) -> list[Erratum]: """Authenticate and order a feed, or raise `FeedError`. - Returns the accepted errata in sequence order. Refuses the whole feed - rather than the offending entry: a feed that equivocates or skips has + Returns the accepted errata in sequence order. Refuses the whole observed + feed rather than the offending entry: one view that conflicts or skips has already failed as a channel, and salvaging the entries an attacker chose to make well formed is not a safe default. """ accepted: list[Erratum] = [] - seen: dict[int, str] = {} + seen: dict[int, tuple[str, str]] = {} previous = last_sequence for erratum in errata: @@ -204,11 +214,15 @@ def verify_feed( "durable memory poisoning, so the feed is refused." ) - if erratum.sequence in seen and seen[erratum.sequence] != erratum.erratum_id: + fingerprint = event_fingerprint(erratum) + if ( + erratum.sequence in seen + and seen[erratum.sequence][1] != fingerprint + ): raise FeedError( - f"sequence {erratum.sequence} conflict: {seen[erratum.sequence]!r} " + f"sequence {erratum.sequence} conflict: {seen[erratum.sequence][0]!r} " f"and {erratum.erratum_id!r} both claim it. The owner has " - "equivocated, or the feed was spliced." + "equivocated within this observed view, or the feed was spliced." ) if erratum.sequence <= previous: @@ -227,7 +241,7 @@ def verify_feed( _check_shape(erratum, roots) - seen[erratum.sequence] = erratum.erratum_id + seen[erratum.sequence] = (erratum.erratum_id, fingerprint) previous = erratum.sequence accepted.append(erratum) diff --git a/prototype/sqlite_store.py b/prototype/sqlite_store.py index bd68b3e..bae8ced 100644 --- a/prototype/sqlite_store.py +++ b/prototype/sqlite_store.py @@ -104,6 +104,20 @@ def enumerate(self, root: str) -> tuple[str, ...]: rows = self._db.execute("SELECT artifact_id FROM artifacts").fetchall() return tuple(sorted(r[0] for r in rows if r[0] in closure)) + def lineage_complete(self, root: str) -> bool: + """Database and write-time ledger inventories agree for this store.""" + + rows = { + row[0] + for row in self._db.execute("SELECT artifact_id FROM artifacts").fetchall() + } + expected = { + artifact.artifact_id + for artifact in self._ledger.artifacts() + if artifact.store == self.name + } + return root in self._ledger.roots_seen() and rows == expected + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: """Commit the gate immediately, in its own transaction. diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 0d6ea50..0fda7a0 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -91,12 +91,13 @@ "!=": operator.ne, } G2_MATRIX_CURRENT_EVIDENCE = ( - "Phase 2 is internally complete: schemas, CLI quarantine checkpoints, semantic probes, " - "key rotation, concurrency, invalid-target, confidentiality, and complete receipt-field " - "binding vectors are implemented; no qualifying independent review is recorded." + "Phase 2 implementation includes conflict-disclosed remediation for split-view " + "equivocation and unsupported empty enumeration, plus schemas, checkpoints, semantic " + "probes, key rotation, invalid-target, confidentiality, and receipt binding; no " + "qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( - "Dated independent external conformance-review result covering the exact complete Phase 2 surface." + "Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation." ) G6_MATRIX_CURRENT_EVIDENCE = ( "No independent report binds an exact commit and deployment to passing " diff --git a/scripts/claim_guard.py b/scripts/claim_guard.py index 56a6f69..be95e5f 100644 --- a/scripts/claim_guard.py +++ b/scripts/claim_guard.py @@ -92,6 +92,16 @@ "Coverage truthfulness requires the signed stores, aggregate, and limitations to match the declared required scope without upgrading missing or opaque evidence.", "coverage truthfulness boundary", ), + ( + "IDEA.md", + "An empty enumeration is not evidence of complete lineage.", + "empty enumeration is not verified coverage", + ), + ( + "IDEA.md", + "A receipt establishes the signed event accepted by one importer.", + "receipt is importer-view evidence", + ), ( "PRIOR_ART.md", "## What would invalidate the claim", diff --git a/spec/README.md b/spec/README.md index 2fbb9f1..c03e89e 100644 --- a/spec/README.md +++ b/spec/README.md @@ -15,6 +15,12 @@ verifies the importer and every signable byte. D2 coverage truthfulness checks that signed stores, aggregate, and limitations match the declared required scope. A correctly signed receipt that overstates coverage passes D1 and fails D2; signature validity never upgrades missing or opaque evidence. + +The reference adapter contract treats an empty enumeration without explicit +root-specific lineage-completeness evidence as `unknown`. Receipts also carry a +signed limitation that importer-local sequencing cannot establish global owner +non-equivocation across split views. The wire coverage vocabulary remains the +four terminal results `verified`, `partial`, `unknown`, and `failed`. | `semantic/probes.json` | Named, strict semantic-probe sets for offline conformance. | | `semantic/verifier-config.json` | Exact synthetic verifier configuration, whose canonical digest binds every observation. | | `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | diff --git a/spec/vectors/protocol-manifest.json b/spec/vectors/protocol-manifest.json index bcb873e..398ee4f 100644 --- a/spec/vectors/protocol-manifest.json +++ b/spec/vectors/protocol-manifest.json @@ -58,6 +58,20 @@ "error_contains": "target" } ], + "split_view_cases": [ + { + "id": "owner-split-view-not-locally-detectable", + "why": "two importers can each receive a valid monotone signed event at sequence 1 while the events contradict; neither importer observes the other view", + "keys": {"owner-v1": "synthetic-owner-v1"}, + "schedule": [{"sequence": 1, "key_id": "owner-v1"}], + "roots": ["mem_01HX"], + "views": [ + [{"erratum_id": "err_view_a", "sequence": 1, "target_root": "mem_01HX", "operation": "supersede", "signer": "owner-v1"}], + [{"erratum_id": "err_view_b", "sequence": 1, "target_root": "mem_01HX", "operation": "correct", "signer": "owner-v1"}] + ], + "outcome": "accept-each-view-with-global-non-equivocation-unproven" + } + ], "confidentiality_case": { "id": "erasure-evidence-content-free", "target_root": "mem_01HX", diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 5c83562..1e090cc 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -32,6 +32,9 @@ def ledger_with_a_mixed_summary() -> LineageLedger: ledger.register_import(ROOT, "fact:diet", store="markdown", content="is vegetarian") ledger.register_import(QUIET, "fact:venue", store="markdown", content="prefers quiet restaurants") ledger.register_import(BUDGET, "fact:budget", store="markdown", content="moderate budget") + ledger.register_import( + "mem_09ZZ", "fact:pet", store="markdown", content="has a cat" + ) ledger.register_derivation( "summary:dining", store="markdown", @@ -63,9 +66,6 @@ def test_the_closure_is_transitive(self) -> None: def test_an_unrelated_artifact_is_not_a_descendant(self) -> None: ledger = ledger_with_a_mixed_summary() - ledger.register_import( - "mem_09ZZ", "fact:pet", store="markdown", content="has a cat" - ) self.assertNotIn("fact:pet", ledger.descendants(ROOT)) def test_still_valid_inputs_exclude_the_retired_root(self) -> None: @@ -91,8 +91,9 @@ def test_quarantine_removes_an_artifact_from_recall(self) -> None: def test_quarantine_does_not_touch_unrelated_artifacts(self) -> None: self.adapter.quarantine(("fact:diet", "summary:dining")) - self.assertEqual( - [item.artifact_id for item in self.adapter.recall("cat")], [] + self.assertIn( + "fact:pet", + [item.artifact_id for item in self.adapter.recall("cat")], ) self.assertIn( "fact:budget", @@ -143,6 +144,15 @@ def setUp(self) -> None: def test_it_enumerates_entries_by_derivation_metadata(self) -> None: self.assertEqual(self.adapter.enumerate(ROOT), ("vec:diet", "vec:dining")) + def test_lineage_completeness_survives_intentional_retirement(self) -> None: + self.assertTrue(self.adapter.lineage_complete(ROOT)) + self.adapter.retire("vec:diet") + self.assertTrue(self.adapter.lineage_complete(ROOT)) + + def test_live_entry_without_source_metadata_breaks_completeness(self) -> None: + self.adapter._text["vec:orphan"] = "untracked live entry" + self.assertFalse(self.adapter.lineage_complete(ROOT)) + def test_quarantined_entries_stop_being_retrieved(self) -> None: self.assertTrue(self.adapter.recall("vegetarian")) self.adapter.quarantine(("vec:diet", "vec:dining")) diff --git a/tests/test_claim_guard.py b/tests/test_claim_guard.py index 327dd3d..f738597 100644 --- a/tests/test_claim_guard.py +++ b/tests/test_claim_guard.py @@ -105,6 +105,30 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("coverage truthfulness boundary", result.stdout) + def test_empty_enumeration_cannot_become_complete_lineage(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "IDEA.md", + "An empty enumeration is not evidence of complete lineage.", + "An empty enumeration proves complete lineage.", + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("empty enumeration", result.stdout) + + def test_receipt_cannot_claim_global_feed_consistency(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "IDEA.md", + "A receipt establishes the signed event accepted by one importer.", + "A receipt establishes the globally consistent owner feed.", + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("importer-view evidence", result.stdout) + def test_inverted_loop_order_is_rejected(self) -> None: def mutate(root: Path) -> None: path = root / "IDEA.md" diff --git a/tests/test_controller.py b/tests/test_controller.py index 1a91d96..e2fefc7 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -27,6 +27,42 @@ OWNER = DemoSigner(b"owner-secret") +class SilentLineageAdapter: + """Enumerable but supplies no evidence that its empty walk is complete.""" + + name = "silent_store" + required = True + + def enumerate(self, root: str) -> tuple[str, ...]: + return () + + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: + return None + + def is_quarantined(self, artifact_id: str) -> bool: + return False + + def coverage(self, root: str) -> Coverage: + return Coverage.VERIFIED + + def dispositions(self, root: str) -> dict[str, str]: + return {} + + +class AuditedEmptyAdapter(SilentLineageAdapter): + name = "audited_empty_store" + + def lineage_complete(self, root: str) -> bool: + return True + + +class SilentFailingAdapter(SilentLineageAdapter): + name = "silent_failing_store" + + def coverage(self, root: str) -> Coverage: + return Coverage.FAILED + + def supersede(sequence: int = 1) -> Erratum: return OWNER.sign_erratum( Erratum( @@ -316,6 +352,67 @@ def test_the_receipt_lists_what_it_could_not_verify(self) -> None: receipt.limitations, ) + def test_empty_enumeration_without_lineage_audit_cannot_verify(self) -> None: + importer = build_importer(OWNER, include_opaque=False) + importer.adapters.append(SilentLineageAdapter()) + receipt = importer.repair(supersede()) + self.assertEqual(receipt.stores["silent_store"], Coverage.UNKNOWN) + self.assertNotEqual(receipt.aggregate, Coverage.VERIFIED) + self.assertTrue( + any("silent_store" in item and "lineage" in item for item in receipt.limitations), + receipt.limitations, + ) + + def test_audited_empty_scope_can_verify(self) -> None: + importer = build_importer(OWNER, include_opaque=False) + importer.adapters.append(AuditedEmptyAdapter()) + receipt = importer.repair(supersede()) + self.assertEqual(receipt.stores["audited_empty_store"], Coverage.VERIFIED) + self.assertEqual(receipt.aggregate, Coverage.VERIFIED) + + def test_missing_lineage_evidence_never_upgrades_a_failure(self) -> None: + importer = build_importer(OWNER, include_opaque=False) + importer.adapters.append(SilentFailingAdapter()) + receipt = importer.repair(supersede()) + self.assertEqual(receipt.stores["silent_failing_store"], Coverage.FAILED) + self.assertEqual(receipt.aggregate, Coverage.FAILED) + + +class SplitViewEquivocationIsOutsideOneImporter(unittest.TestCase): + def test_one_importer_remembers_a_conflict_across_observe_calls(self) -> None: + importer = build_importer(OWNER, include_opaque=False) + first_event = supersede().replace(erratum_id="err_A", signature=None) + second_event = supersede().replace( + erratum_id="err_B", replacement="is vegan now", signature=None + ) + importer.quarantine(OWNER.sign_erratum(first_event)) + with self.assertRaisesRegex(Exception, "conflict in this importer view"): + importer.quarantine(OWNER.sign_erratum(second_event)) + + def test_each_receipt_discloses_that_its_feed_view_is_local(self) -> None: + first_event = supersede().replace(erratum_id="err_A", signature=None) + second_event = supersede().replace( + erratum_id="err_B", + replacement="is vegan now", + postconditions={ + "negative": "vegetarian", + "positive": "is vegan now", + "preserve": "quiet restaurants|moderate budget", + }, + signature=None, + ) + first = build_importer(OWNER).repair(OWNER.sign_erratum(first_event)) + second = build_importer(OWNER).repair(OWNER.sign_erratum(second_event)) + + self.assertTrue(first.verify(build_importer(OWNER).signer.public)) + self.assertTrue(second.verify(build_importer(OWNER).signer.public)) + self.assertNotEqual(first.erratum_id, second.erratum_id) + for receipt in (first, second): + self.assertTrue( + any("global" in item and "equivocation" in item for item in receipt.limitations), + receipt.limitations, + ) + class FeedRollbackIsRefusedByTheController(unittest.TestCase): def test_an_older_erratum_after_a_newer_one_is_refused(self) -> None: diff --git a/tests/test_errata_feed.py b/tests/test_errata_feed.py index 98e1806..4222e6b 100644 --- a/tests/test_errata_feed.py +++ b/tests/test_errata_feed.py @@ -126,6 +126,14 @@ def test_two_different_events_at_one_sequence_are_rejected(self) -> None: verify_feed([first, equivocation], owner=OWNER.public, roots=ROOTS) self.assertIn("conflict", str(raised.exception)) + def test_same_id_with_different_signed_content_is_also_a_conflict(self) -> None: + first = OWNER.sign_erratum(erratum(1, erratum_id="err_same")) + second = OWNER.sign_erratum( + erratum(1, erratum_id="err_same", replacement="different state") + ) + with self.assertRaisesRegex(FeedError, "conflict"): + verify_feed([first, second], owner=OWNER.public, roots=ROOTS) + class FeedRejectsAmbiguousOrIllFormedTargets(unittest.TestCase): def test_an_unregistered_target_root_is_rejected(self) -> None: diff --git a/tests/test_readiness.py b/tests/test_readiness.py index b33df36..620a2b4 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -435,7 +435,7 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Phase 2 is internally complete: schemas, CLI quarantine checkpoints, semantic probes, key rotation, concurrency, invalid-target, confidentiality, and complete receipt-field binding vectors are implemented; no qualifying independent review is recorded.", + "Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation and unsupported empty enumeration, plus schemas, checkpoints, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded.", "local tests prove readiness.", ) @@ -454,7 +454,7 @@ def test_g2_matrix_next_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Dated independent external conformance-review result covering the exact complete Phase 2 surface.", + "Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation.", "Local tests are enough.", ) diff --git a/tests/test_schema.py b/tests/test_schema.py index d4befcb..c708676 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -211,6 +211,35 @@ def test_feed_vectors_execute_named_stateful_rules(self) -> None: events, owner=schedule, roots=RootRegistry(case["roots"]) ) + def test_split_view_vector_accepts_each_view_but_proves_no_global_consistency(self) -> None: + for case in self.protocol["split_view_cases"]: + with self.subTest(case=case["id"]): + signers = { + key_id: DemoSigner(seed.encode(), key_id=key_id) + for key_id, seed in case["keys"].items() + } + schedule = OwnerKeySchedule(tuple( + (activation["sequence"], signers[activation["key_id"]].public) + for activation in case["schedule"] + )) + accepted_views = [ + verify_feed( + [self._event(record, signers) for record in view], + owner=schedule, + roots=RootRegistry(case["roots"]), + ) + for view in case["views"] + ] + self.assertEqual([len(view) for view in accepted_views], [1, 1]) + self.assertNotEqual( + accepted_views[0][0].erratum_id, + accepted_views[1][0].erratum_id, + ) + self.assertEqual( + case["outcome"], + "accept-each-view-with-global-non-equivocation-unproven", + ) + def test_confidentiality_vector_keeps_forbidden_value_out_of_evidence(self) -> None: case = self.protocol["confidentiality_case"] importer = build_importer(OWNER) From b45e4ba54fece9e29f578072f483847a65e12ce4 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 14:17:15 +0200 Subject: [PATCH 57/88] docs: publish feedback remediation target --- docs/PUBLICATION_LOG.md | 32 +++++++++--- publication/active-surfaces.json | 87 +++++++------------------------- scripts/check_publication.py | 40 +++------------ tests/test_publication.py | 12 +++-- 4 files changed, 58 insertions(+), 113 deletions(-) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index e9846b5..51796f7 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -24,9 +24,13 @@ readiness gate. Last verified: 2026-08-13. | GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | | Historical Phase 2 review target | Commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2), digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` | Stale: this target predated the accepted prior-art corrections and must not be used for a current review. Preserved as historical publication evidence. | | Corrected complete Phase 2 review target | Commit [`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878), digest `03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14` | Complete corrected specification, prior-art, licensing, and cryptographic-qualification surface. Checkout and committed-source digests match; exact-target CI passed Python 3.11 and 3.13. Internal evidence does not satisfy G2. | +| Current feedback-remediation target | Commit [`a477fe4f5c86730031b6285d9505778fb8eec060`](https://github.com/thomaswillner/llm-errata/commit/a477fe4f5c86730031b6285d9505778fb8eec060), digest `a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3` | Remediates same-importer conflict persistence, signed split-view limitations, and unsupported empty enumeration. Checkout and committed-tree digests match; exact-target [CI run 31698542878](https://github.com/thomaswillner/llm-errata/actions/runs/31698542878) passed Python 3.11 and 3.13. Internal evidence does not satisfy G2. | | Interested-party technical review | [Two counterexamples](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5276848751) | Bound to the corrected target with explicit Inspeximus/market conflict. Reproduced same-importer versus split-view equivocation limits and unsupported empty-enumeration success. Useful external technical evidence, but not a qualifying complete independent G2 review. | | Cross-project remediation and adapter offer | [Inspeximus follow-up](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5279081884) | Inspeximus source commit `3661102` fixes an adjacent known-hole audit defect and offers an independently authored adapter plus benchmark harness with explicit commercial interest. No adapter artifact exists yet; the producer cannot also validate its own adapter for G4. | | Cryptography maintainer response | [Pure-Python Ed25519 reference](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5275036522) | Useful referral and evidence that the outreach lacked plain-language context. The linked pure-Python project is a comparison oracle, not audited constant-time G3 evidence. | +| Human cryptography/context reply | [Plain-language response](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5280206599) | Read back from GitHub with author and body verified. Explains the project and AI-assistance boundary, records the comparison oracle, and states why G3 remains blocked. Technical response only; no further review request. | +| Human findings and adapter reply | [Point-by-point remediation response](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5280210050) | Read back from GitHub with author and body verified. Accepts both reproduced findings, explains the `unknown` decision, records the split-view limitation, and defines a clean-room Inspeximus adapter plus separate-validator shape. Recruitment-only for G4; conflict disclosure remains controlling. | +| Current PR remediation checkpoint | [PR #8 feedback checkpoint](https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5280225709) | Read back from GitHub with author and body verified. Publishes current commit/digest, exact-head CI, reply URLs, and unchanged G2–G6 blockers without additional mentions. Publication only; it supersedes prior active target pointers. | | Final independent-implementation call | [Issue #5 completion update](https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5272475120) | Publishes the immutable Phase 2 commit and digest for two independent adapters and a separate validator. No implementation has been accepted; G4 remains `BLOCKED`. | | Final three-system nomination call | [Issue #6 completion update](https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5272475614) | Publishes the immutable Phase 2 target and exact nomination fields. No system authorization or experiment evidence exists; G5 remains `BLOCKED`. | | Interested-party review follow-up | [Inspeximus maintainer reply](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5272476094) | Acknowledges the source-grounded correction and points to the final review target. Conflict disclosure remains controlling; a future review from this maintainer cannot satisfy G2 alone. | @@ -39,11 +43,21 @@ readiness gate. Last verified: 2026-08-13. | Phase 2 completion announcement | [Discussion #9 completion update](https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648) | Announces internal completion and routes falsifiers to Issues #4, #5, #6, and #10. Publication only; it records no external acceptance or readiness-gate change. | | Six-step readiness checkpoint | [PR #8 checkpoint](https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272526975) | Audits every approved production-readiness step against exact internal and external evidence. Records `NOT_PROD_READY`, the external blockers, PR #3 status, validation, and publication boundaries without claiming certification. | -One targeted invitation now has a conflict-disclosed technical review and -implementation offer recorded above; it does not satisfy G2 or G4 by itself. -The corrected follow-ups use six role-specific mentions across three issues and -no mentions in PR #8 or Discussion #9. No produced adapter, separate validator, -operated-system approval, or CODEOWNER volunteer is recorded. +One current-target invitation now asks the conflict-disclosed Inspeximus +maintainer for an independently authored adapter and harness. It does not +satisfy G2 or G4 by itself. Prior role-specific calls remain historical and +visible, but their target is superseded by the current PR checkpoint. No +produced adapter, separate validator, operated-system approval, or CODEOWNER +volunteer is recorded. + +The current complete conformance review target is source commit +[`a477fe4f5c86730031b6285d9505778fb8eec060`](https://github.com/thomaswillner/llm-errata/commit/a477fe4f5c86730031b6285d9505778fb8eec060) +with canonical Phase 2 surface digest +`a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3`. +The digest was recomputed from checkout and committed tree with an exact match. +The exact-target [GitHub Actions run +31698542878](https://github.com/thomaswillner/llm-errata/actions/runs/31698542878) +passed Python 3.11 and Python 3.13. The former complete conformance review target is source commit [`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878) @@ -53,8 +67,8 @@ The digest was recomputed both from the checkout and from `git show` at that commit with an exact match. The exact-target [GitHub Actions run 31646616085](https://github.com/thomaswillner/llm-errata/actions/runs/31646616085) passed Python 3.11 and Python 3.13. It is now historical because external review -found two accepted defects in that surface; a remediation target will supersede -it after the corrected source is committed, digested, tested, and published. +found two accepted defects in that surface; the current remediation target +supersedes it. The previously frozen source commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2) and digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` @@ -116,5 +130,7 @@ GitHub calls, invitations, CI results, maintainer work, and conflict-disclosed interested-party review are not qualifying independent evidence. Phase 2 implementation now includes remediation for same-view versus split-view equivocation and unsupported empty enumeration. G2 remains `BLOCKED` pending a -dated qualifying independent review of the exact post-remediation target. +dated qualifying independent review of commit +`a477fe4f5c86730031b6285d9505778fb8eec060` and digest +`a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3`. Repository verdict remains `NOT_PROD_READY`. diff --git a/publication/active-surfaces.json b/publication/active-surfaces.json index 04bf5db..d47b015 100644 --- a/publication/active-surfaces.json +++ b/publication/active-surfaces.json @@ -1,8 +1,8 @@ { "schema_version": 1, "review_target": { - "commit": "08b95263c9ed700c43aea0b285696956cc23e878", - "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14" + "commit": "a477fe4f5c86730031b6285d9505778fb8eec060", + "surface_digest": "a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3" }, "license": { "specification_implementation": "irrevocable worldwide royalty-free commercial and non-commercial", @@ -16,87 +16,36 @@ }, "surfaces": [ { - "id": "g2-g3-targeted-review", + "id": "g4-inspeximus-adapter-call", "kind": "issue-comment", - "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5274367774", + "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5280210050", "published": "2026-08-13", - "commit": "08b95263c9ed700c43aea0b285696956cc23e878", - "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", - "gates": ["G2", "G3"], - "roles": ["protocol-reviewer", "cryptography-reviewer-or-referrer"], - "mentions": ["santhoshravindran7", "jedisct1"], - "supersedes": [ - "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269378077", - "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269524323", - "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5269850937", - "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5272345570" - ], - "evidence_boundary": "recruitment-only" - }, - { - "id": "g4-targeted-implementation", - "kind": "issue-comment", - "url": "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5274369945", - "published": "2026-08-13", - "commit": "08b95263c9ed700c43aea0b285696956cc23e878", - "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", + "commit": "a477fe4f5c86730031b6285d9505778fb8eec060", + "surface_digest": "a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3", "gates": ["G4"], - "roles": ["adapter-author", "validator-author"], - "mentions": ["joshuaswarren", "alphaonedev"], + "roles": ["inspeximus-adapter-author"], + "mentions": ["DanceNitra"], "supersedes": [ - "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269378246", - "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269523953", - "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5269524142", - "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5272475120" + "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5274369945" ], "evidence_boundary": "recruitment-only" }, { - "id": "g5-targeted-systems", - "kind": "issue-comment", - "url": "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5274371747", - "published": "2026-08-13", - "commit": "08b95263c9ed700c43aea0b285696956cc23e878", - "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", - "gates": ["G5"], - "roles": ["mem0-system-operator", "cognee-system-operator"], - "mentions": ["kartik-mem0", "Vasilije1990"], - "supersedes": [ - "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269378440", - "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269537990", - "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5269538156", - "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5272475614" - ], - "evidence_boundary": "recruitment-only" - }, - { - "id": "pr8-correction", + "id": "pr8-feedback-remediation", "kind": "pull-request-comment", - "url": "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5274373337", - "published": "2026-08-13", - "commit": "08b95263c9ed700c43aea0b285696956cc23e878", - "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", - "gates": ["G2", "G3", "G4", "G5", "G6"], - "roles": [], - "mentions": [], - "supersedes": [ - "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272345782", - "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272526975" - ], - "evidence_boundary": "publication-only" - }, - { - "id": "discussion9-correction", - "kind": "discussion-comment", - "url": "https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17995307", + "url": "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5280225709", "published": "2026-08-13", - "commit": "08b95263c9ed700c43aea0b285696956cc23e878", - "surface_digest": "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", + "commit": "a477fe4f5c86730031b6285d9505778fb8eec060", + "surface_digest": "a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3", "gates": ["G2", "G3", "G4", "G5", "G6"], "roles": [], "mentions": [], "supersedes": [ - "https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648" + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5274367774", + "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5274369945", + "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5274371747", + "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5274373337", + "https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17995307" ], "evidence_boundary": "publication-only" } diff --git a/scripts/check_publication.py b/scripts/check_publication.py index 87acc9e..1292477 100644 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -14,49 +14,25 @@ ROOT = Path(__file__).resolve().parents[1] MANIFEST = ROOT / "publication" / "active-surfaces.json" -CANONICAL_COMMIT = "08b95263c9ed700c43aea0b285696956cc23e878" +CANONICAL_COMMIT = "a477fe4f5c86730031b6285d9505778fb8eec060" CANONICAL_DIGEST = ( - "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14" + "a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3" ) REPOSITORY_URL = "https://github.com/thomaswillner/llm-errata" ALLOWED_GATES = {"G2", "G3", "G4", "G5", "G6"} REQUIRED_ATTRIBUTION = {"LLM Errata", "Thomas Willner", REPOSITORY_URL} REQUIRED_SURFACES = { - "g2-g3-targeted-review": { + "g4-inspeximus-adapter-call": { "kind": "issue-comment", - "url": f"{REPOSITORY_URL}/issues/4#issuecomment-5274367774", - "gates": ["G2", "G3"], - "roles": ["protocol-reviewer", "cryptography-reviewer-or-referrer"], - "mentions": ["santhoshravindran7", "jedisct1"], - "evidence_boundary": "recruitment-only", - }, - "g4-targeted-implementation": { - "kind": "issue-comment", - "url": f"{REPOSITORY_URL}/issues/5#issuecomment-5274369945", + "url": f"{REPOSITORY_URL}/issues/4#issuecomment-5280210050", "gates": ["G4"], - "roles": ["adapter-author", "validator-author"], - "mentions": ["joshuaswarren", "alphaonedev"], - "evidence_boundary": "recruitment-only", - }, - "g5-targeted-systems": { - "kind": "issue-comment", - "url": f"{REPOSITORY_URL}/issues/6#issuecomment-5274371747", - "gates": ["G5"], - "roles": ["mem0-system-operator", "cognee-system-operator"], - "mentions": ["kartik-mem0", "Vasilije1990"], + "roles": ["inspeximus-adapter-author"], + "mentions": ["DanceNitra"], "evidence_boundary": "recruitment-only", }, - "pr8-correction": { + "pr8-feedback-remediation": { "kind": "pull-request-comment", - "url": f"{REPOSITORY_URL}/pull/8#issuecomment-5274373337", - "gates": ["G2", "G3", "G4", "G5", "G6"], - "roles": [], - "mentions": [], - "evidence_boundary": "publication-only", - }, - "discussion9-correction": { - "kind": "discussion-comment", - "url": f"{REPOSITORY_URL}/discussions/9#discussioncomment-17995307", + "url": f"{REPOSITORY_URL}/pull/8#issuecomment-5280225709", "gates": ["G2", "G3", "G4", "G5", "G6"], "roles": [], "mentions": [], diff --git a/tests/test_publication.py b/tests/test_publication.py index b79620f..de7e8d1 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -37,7 +37,7 @@ def mutate(root: Path) -> None: def test_stale_commit_is_rejected(self) -> None: self._assert_manifest_mutation_is_rejected( lambda payload: payload["review_target"].__setitem__( - "commit", "50e895fbfec544b16c94caa07bf2d1f4049a42e2" + "commit", "08b95263c9ed700c43aea0b285696956cc23e878" ), "review target", ) @@ -46,7 +46,7 @@ def test_stale_digest_is_rejected(self) -> None: self._assert_manifest_mutation_is_rejected( lambda payload: payload["review_target"].__setitem__( "surface_digest", - "9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804", + "03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14", ), "review target", ) @@ -96,13 +96,17 @@ def mutate(payload: dict[str, object]) -> None: def test_duplicate_evidence_role_is_rejected(self) -> None: def mutate(payload: dict[str, object]) -> None: - payload["surfaces"][1]["roles"][0] = payload["surfaces"][0]["roles"][0] + payload["surfaces"][1]["roles"] = [ + payload["surfaces"][0]["roles"][0] + ] self._assert_manifest_mutation_is_rejected(mutate, "unique evidence roles") def test_duplicate_mention_is_rejected(self) -> None: def mutate(payload: dict[str, object]) -> None: - payload["surfaces"][1]["mentions"][0] = payload["surfaces"][0]["mentions"][0] + payload["surfaces"][1]["mentions"] = [ + payload["surfaces"][0]["mentions"][0] + ] self._assert_manifest_mutation_is_rejected(mutate, "unique GitHub mentions") From ac4468faf73c2cc7949dd29b2a2a151f5bd23116 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 17:05:35 +0200 Subject: [PATCH 58/88] fix(adapter): preserve checkpoint coverage contract --- AGENTS.md | 5 + CHANGELOG.md | 12 ++ CONTRIBUTING.md | 2 + IDEA.md | 4 + INDEPENDENT_IMPLEMENTATION.md | 36 ++++++ PRIOR_ART.md | 16 +++ PRODUCTION_READINESS.md | 4 +- README.md | 8 +- REVIEW_REQUEST.md | 5 + ROADMAP.md | 7 +- SOURCES.md | 2 +- prototype/README.md | 12 +- prototype/adapters.py | 77 ++++++++++++- prototype/controller.py | 99 ++++++++++++---- prototype/sqlite_store.py | 24 ++++ prototype/strategies.py | 24 ++-- scripts/check_readiness.py | 6 +- scripts/claim_guard.py | 10 ++ spec/README.md | 16 ++- tests/test_adapters.py | 21 ++++ tests/test_claim_guard.py | 28 +++++ tests/test_controller.py | 206 +++++++++++++++++++++++++++++++++- tests/test_readiness.py | 2 +- 23 files changed, 569 insertions(+), 57 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 91cc4da..c83f2b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,6 +126,11 @@ Never convert missing lineage, inaccessible stores, skipped tests, timeouts, or - Verify receipt binding to the erratum and relevant pre-repair and post-repair state. - Keep deterministic structural checks separate from model-graded behavioral probes. - Never mark an opaque required store green merely because accessible stores passed. +- Never hardcode checkpoint success from enumeration. Preserve adapter-supplied + quarantine-phase coverage and carry a worse checkpoint result into the receipt. +- Keep the declared adapter protocol synchronized with every controller and + strategy call; independent adapters must own their repair inputs rather than + depending silently on reference-ledger registration. ## Required completion check diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c821a1..13571b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ Changes to the bounded novelty statement or the source comparison are recorded h ### Added — production-readiness evidence +- Durable quarantine checkpoints now preserve adapter-supplied phase coverage + instead of hardcoding every enumerable adapter to `verified`; a later final + success cannot erase an earlier `partial`, `unknown`, or `failed` result. +- `StoreAdapter` now declares the complete controller and repair call surface, + including quarantine coverage, source mapping, store-owned repair inputs, + retirement, rebuild, and recall. Independent adapters no longer have to copy + their lineage into the reference `LineageLedger`. +- Recorded Inspeximus `v2.7.0` as a tagged external adapter candidate with its + preserved v2.6.1 contamination disclosure and claimed clean-room rewrite. It + targets historical LLM Errata commit `a477fe4` and remains unvalidated G4 + evidence until provenance review and current-target rebinding are complete. + - Corrected two conflict-disclosed external findings. Importers now remember same-view sequence conflicts across separate observe calls, while receipts explicitly disclaim global non-equivocation across split views. Required diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3949925..18490cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,6 +47,8 @@ A conformance change should state: - observable inputs, outputs, and state transitions; - failure, timeout, replay, partial-coverage, and unobservable-store behavior; - empty enumeration with and without root-specific lineage-completeness evidence; +- adapter-supplied quarantine-phase coverage separately from final repair coverage; +- store-owned source mapping and repair inputs without hidden reference-ledger coupling; - same-importer conflicts separately from owner split views across importers; - security and privacy consequences; - positive, negative, and preservation tests; diff --git a/IDEA.md b/IDEA.md index 75cbab0..c461649 100644 --- a/IDEA.md +++ b/IDEA.md @@ -265,6 +265,10 @@ may report `verified` for an empty root scope only when it also establishes a root-specific write-time or audited lineage authority; otherwise the signed store result is `unknown` with a limitation. +A durable checkpoint records adapter-supplied quarantine coverage; final repair cannot upgrade a worse checkpoint result. + +The published adapter contract must expose every controller and repair operation without hidden reference-ledger dependencies. + A receipt establishes the signed event accepted by one importer. Importer-local sequence checks detect conflicts visible in that view, but cannot establish global owner non-equivocation across split views without an external witnessed diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index 962b7ed..9d3a3f8 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -30,6 +30,36 @@ Returning an empty artifact list is not enough. If the implementation cannot show that its enumeration authority was complete for the root, it must report `unknown` and bind the limitation into its receipt. +## Complete adapter contract + +The controller calls the following surface; implementing only enumeration and +coverage is insufficient: + +| Method | Required meaning | +|---|---| +| `enumerate(root)` | Return every store artifact reachable from the root through declared lineage. | +| `lineage_complete(root)` | State whether that root-specific walk is complete. Only exact `True` permits verified coverage. | +| `quarantine(ids)` / `is_quarantined(id)` | Gate affected recall, then prove every enumerated artifact remains gated. | +| `quarantine_coverage(root)` | Report `verified`, `partial`, `unknown`, or `failed` for the durable pre-repair checkpoint. This is distinct from final repair coverage. | +| `source_artifact(id)` | Map a store record to its stable lineage node. Return the ID unchanged when they are identical. | +| `repair_inputs(id)` | Return direct store-owned lineage inputs. The adapter need not copy its records into the reference `LineageLedger`. | +| `retire(id, superseded_at=None)` | Retire an invalid root artifact; preserve history only when `superseded_at` is supplied. | +| `rebuild(id, *, inputs, replacement)` | Reconstruct a mixed artifact from surviving inputs plus any replacement. | +| `recall(term)` | Return active hits with a string `.content` field for negative, positive, and preservation probes. | +| `snapshot()` | Return inspectable adapter state for checkpoint drift detection and receipt pre/post-state binding. Missing state binding prevents verified coverage. | +| `coverage(root)` / `dispositions(root)` | Report final post-repair coverage and one disposition for every enumerated artifact. | + +Checkpoint coverage is signed evidence, not scratch bookkeeping. A later +`verified` result cannot erase an earlier `partial`, `unknown`, or `failed` +checkpoint. Missing, raising, or malformed quarantine-coverage evidence fails +closed as `unknown`. + +Keep independent propositions as independently addressable lineage artifacts. +If one store record combines the corrected proposition with unrelated facts, +the adapter cannot retire one input while preserving the others without an +additional decomposition or rebuild contract; it must report that limit rather +than fabricate complete repair. + ## Independence and evidence An implementation report must name its authors, repository and commit, supported @@ -37,6 +67,12 @@ specification version, dependencies, test commands, unsupported behavior, and licence. Shared conformance vectors are expected. Shared reference-adapter code disqualifies the implementation as independent evidence. +If an earlier version was written from reference code, preserve that history, +identify the affected version, and disclose the clean-room rewrite. A rewrite +may become a candidate independent implementation only after separate review of +its provenance and behavior; author testimony and line-overlap counts are useful +evidence but are not third-party validation. + One producer may supply an adapter and benchmark results, but that producer cannot also occupy the separately authored third-party validator role for its own implementation. Commercial interest and other conflicts must be disclosed; diff --git a/PRIOR_ART.md b/PRIOR_ART.md index a6b6359..50c32d8 100644 --- a/PRIOR_ART.md +++ b/PRIOR_ART.md @@ -114,6 +114,22 @@ own adapter or store instrumentation and neither result is independent certification. The maintainer's 0.0000 dogfood ratio remains interested-party testimony, not independently reproduced evidence. +The Inspeximus maintainer then published a tagged adapter candidate at +[`v2.7.0`](https://github.com/DanceNitra/inspeximus/tree/v2.7.0), commit +`ccdb30de5a32896fafcb4ed18a7a6f228691e078`. Its preserved history and source +header disclose that v2.6.1 was written with this repository's reference adapter +open and therefore does not qualify as independent evidence; the maintainer says +v2.7.0 was rewritten from the protocol signature and prose contract. That +externally authored candidate confirmed the empty-lineage remediation against +historical target `a477fe4f5c86730031b6285d9505778fb8eec060`, then exposed two further reference +defects: checkpoint coverage had been hardcoded for enumerable adapters, and the +published adapter interface omitted repair methods and a hidden reference-ledger +dependency. Those findings are accepted and remediated. The v2.7.0 adapter is a +candidate external implementation, not established independent evidence or a G4 +pass: its clean-room provenance and behavior still require separate validation, +its producer cannot validate its own adapter, and it must rebind to the current +immutable target before current conformance results can be evaluated. + These mechanisms apply to explicitly keyed or successfully extractor-keyed assertions and recorded lineage. The documented scope is one store, not every vector index, prompt log, backup, or independently operated importer. diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index a71b2d5..30696f2 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -15,9 +15,9 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation and unsupported empty enumeration, plus schemas, checkpoints, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | -| G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | `ROADMAP.md` records this dependency; no independent implementations or validator result are recorded. | Dated evidence from two independent adapters and separately produced third-party validator result. | +| G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | Inspeximus `v2.7.0` is one tagged externally authored adapter candidate with disclosed v2.6.1 reference-code contamination and a claimed clean-room rewrite. It targets historical commit `a477fe4f5c86730031b6285d9505778fb8eec060`; provenance, current-target behavior, a second candidate, and a third-party validator result remain unverified. | Rebind candidates to the current immutable target; obtain dated evidence from two independently authored adapters, including separate provenance review where needed, and a separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | | G6 | All ten operational scopes pass from one independent report bound to exact commit and deployment, with declared thresholds and measured comparators. | `BLOCKED` | No independent report binds an exact commit and deployment to passing measured comparators for all ten operational scopes. | One qualifying independent report with declared workload, platform, failure domain, observation window, numeric thresholds, raw artifacts, and passing measurements for every scope. | diff --git a/README.md b/README.md index 416f6d7..e00561d 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,12 @@ store remains `unknown`. A receipt also speaks only for the signed feed view that one importer accepted. Detecting different views delivered to different importers requires an external witness or append-only transparency mechanism. +A durable quarantine checkpoint records the adapter's own phase-specific +coverage rather than assuming every enumerable store is verified. Final repair +cannot erase an earlier `partial`, `unknown`, or `failed` checkpoint. Independent +adapters own their lineage inputs behind the published interface; they do not +have to mirror store internals into the reference ledger. + ## The repair triad Every repair is evaluated against three postconditions: @@ -173,7 +179,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface are implemented with two conflict-disclosed external findings remediated: split-view equivocation is now stated as outside importer-local proof, and empty enumeration cannot become verified without root-specific lineage-completeness evidence. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, and mutation coverage for every signed receipt field. G2 remains `BLOCKED`: the reviewer disclosed a commercial conflict and did not cover the full required conformance scope, so a qualifying independent review is still absent. +Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface include conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, and mutation coverage for every signed receipt field. G2 remains `BLOCKED`: the reviewer disclosed a commercial conflict and did not cover the full required conformance scope. G4 also remains `BLOCKED`: one tagged external adapter candidate exists, but its clean-room provenance and behavior are not third-party validated and no separate validator result exists. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/REVIEW_REQUEST.md b/REVIEW_REQUEST.md index 4a81cbb..f02d06f 100644 --- a/REVIEW_REQUEST.md +++ b/REVIEW_REQUEST.md @@ -34,6 +34,11 @@ from global non-equivocation. A receipt proves which signed event one importer accepted; it does not prove every importer received the same event. Also test whether an empty adapter enumeration has root-specific lineage-completeness evidence. Empty-without-evidence must be `unknown`, not `verified`. +Also verify that a durable checkpoint records the adapter's own quarantine- +phase coverage and that final repair cannot overwrite an earlier `partial`, +`unknown`, or `failed` result. Build one adapter using only the published +contract: it must not discover `retire`, `rebuild`, `recall`, source mapping, or +lineage-input requirements through exceptions or reference-ledger internals. ## Required review record diff --git a/ROADMAP.md b/ROADMAP.md index c127d38..8943ca2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -124,18 +124,19 @@ Only after the file-backed proof passes: errata audit ``` -4. Define an adapter interface for enumeration, root-specific lineage-completeness evidence, quarantine, reconstruction, verification, and coverage reporting. Empty enumeration without that evidence is `unknown`. +4. Define the complete adapter interface for enumeration, root-specific lineage-completeness evidence, quarantine and quarantine-phase coverage, store-owned repair inputs, retirement, reconstruction, recall probes, final dispositions, and coverage reporting. Empty enumeration or missing phase evidence is `unknown`; final success cannot overwrite a worse durable checkpoint. 5. Publish conformance vectors for signatures, sequencing, key rotation, concurrent events, invalid targets, receipt binding, and confidentiality. 6. Add model-assisted semantic probes behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. **Implemented internally:** [`prototype/semantic.py`](prototype/semantic.py), [`spec/semantic/`](spec/semantic/), and `errata semantic-test` record only structured, configuration-bound observations; inconclusive, malformed, missing, duplicate, or drifted required evidence is not success. The CLI quarantine checkpoint is also implemented internally. `errata quarantine` authenticates exactly the next pending erratum, gates enumerable descendants, records opaque stores as `unknown`, and atomically persists a digest bound to -the erratum, sequence, target, pre-state, adapter inventory, and gated set. +the erratum, sequence, target, pre-state, adapter inventory, gated set, and each +adapter's own quarantine-phase coverage. `errata repair` re-authenticates and refuses missing, consumed, replayed, or drifted checkpoints before rebuild. -Phase 2 internal implementation is complete. G2 remains blocked until a dated independent external reviewer evaluates the exact committed surface; local tests and author-directed review cannot satisfy that gate. Broader interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). +Phase 2 internal implementation is complete. G2 remains blocked until a dated independent external reviewer evaluates the exact committed surface; local tests and author-directed review cannot satisfy that gate. Inspeximus `v2.7.0` is one tagged externally authored adapter candidate with a claimed clean-room rewrite, not established independent evidence. It targets historical commit `a477fe4f5c86730031b6285d9505778fb8eec060`; provenance review and rebinding to the current immutable target are required before current conformance results can be evaluated. Broader interoperability remains contingent on two independently implemented adapters consuming the same erratum and a third-party validator evaluating their receipts consistently. See [REVIEW_REQUEST.md](REVIEW_REQUEST.md) and [INDEPENDENT_IMPLEMENTATION.md](INDEPENDENT_IMPLEMENTATION.md). ## Phase 3 — Interoperability experiment diff --git a/SOURCES.md b/SOURCES.md index 23c7ed0..5436c64 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -48,7 +48,7 @@ the default branch and will move. | Source | Pin | Verified 2026-08-07 | |---|---|---| -| [Inspeximus](https://github.com/DanceNitra/inspeximus) | `4c711f2982911841d86d7ac1989b0ffb866dc891` (review baseline, 2026-08-12); remediation `36611027a463a8e526e23baf2d6bb8d9797b67ac` (2026-08-13) | Baseline confirms `retract_lineage`, `include_superseded`, `needs_rederivation`, and `rederive`. Remediation source sets package version 2.6.0, makes known unresolved derivation holes `partially_audited`, reports `subject_reachable_records`, and includes negative controls explaining why a global declared-ratio threshold would be wrong. No public GitHub release/tag was found, so 2.6.0 is source-version evidence, not a verified release artifact. Both commits post-date the 2026-08-01 cutoff and are supplementary collision evidence. | +| [Inspeximus](https://github.com/DanceNitra/inspeximus) | `4c711f2982911841d86d7ac1989b0ffb866dc891` (review baseline, 2026-08-12); `v2.6.0` / `36611027a463a8e526e23baf2d6bb8d9797b67ac`; `v2.7.0` / `ccdb30de5a32896fafcb4ed18a7a6f228691e078` (2026-08-13) | Baseline confirms `retract_lineage`, `include_superseded`, `needs_rederivation`, and `rederive`. v2.6.0 makes known unresolved derivation holes `partially_audited` and reports `subject_reachable_records`. v2.7.0 adds a tagged LLM Errata adapter candidate and preserves explicit provenance disclosure: v2.6.1 copied reference structure and is disqualified; v2.7.0 claims a clean-room rewrite and reports adapter-contract defects reproduced locally here. Tags and source version are verified; GitHub Releases has no v2.6.0–v2.7.0 Release entries. The adapter targets historical LLM Errata commit `a477fe4f5c86730031b6285d9505778fb8eec060` and must rebind to the current immutable target. Candidate implementation and interested-party evidence only, not established independence, third-party validation, or G4 completion. | | [jedisct1/ed25519.py](https://github.com/jedisct1/ed25519.py) | `67902d339ea47418a60fb7684255b81bc4f6d46e` (2026-08-13) | Maintainer-provided pure-Python Ed25519 comparison implementation. README claims RFC 8032 vectors, canonical/small-order rejection, batch verification, and optional randomized signing countermeasures. Repository supplies no detected licence file or independent audit/build qualification, so it is a comparison oracle only and does not satisfy G3. | | [Doyle, A Truth Maintenance System](https://dblp.org/rec/journals/ai/Doyle79.html) | Artificial Intelligence 12(3), 1979; DBLP `journals/ai/Doyle79` | Stable bibliographic record confirms title and publication. Cited by Inspeximus as prior retract-and-retain lineage. Included to make historical dependency-retraction collision explicit; LLM Errata claims no invention of this mechanism. | | [IAB Data Deletion Request Framework](https://github.com/InteractiveAdvertisingBureau/Data-Subject-Rights/blob/main/Data%20Deletion%20Request%20Framework.md) | `b9418f5394ca91193181a61c567ffbbdff79cdef` (2025-02-06) | Repository unchanged since well before the cutoff | diff --git a/prototype/README.md b/prototype/README.md index f123c1c..f84e8ef 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -79,10 +79,11 @@ A full lifecycle without importing Python: `init`, `export`, `derive`, `verify`. `quarantine` authenticates exactly the next pending erratum, gates every -enumerable descendant, records opaque stores as `unknown`, and atomically +enumerable descendant, records each adapter's own quarantine-phase coverage, +records opaque or missing checkpoint evidence as `unknown`, and atomically writes `checkpoints/-.json`. Its canonical digest binds the erratum, target, inspectable pre-state root, adapter inventory, limitations, -and gated artifact set. `repair` re-authenticates and refuses missing, mutated, +gated artifact set, and reported checkpoint coverage. `repair` re-authenticates and refuses missing, mutated, consumed, wrong-target, state-drifted, adapter-drifted, or ungated evidence. Consumption happens only after receipt and applied-state writeback, so an interrupted rebuild retains an unconsumed checkpoint for safe resume. @@ -94,6 +95,13 @@ required store `unknown`, even when enumeration returns an empty tuple and the adapter's own coverage method claims `verified`. This is an adapter attestation, not independent proof against a dishonest store. +Checkpoint coverage and final repair coverage are different observations. The +adapter supplies both through `quarantine_coverage(root)` and `coverage(root)`; +the controller carries the worse result into the signed receipt, so a later +success cannot erase an earlier partial or failed gate. Repair planning also +uses adapter-owned `source_artifact(id)` and `repair_inputs(id)` rather than +requiring independent store records to be copied into the reference ledger. + Exit codes are part of the interface. `0` is success, `1` is a refusal or a failed check, and **`2` means the repair ran and the result is not verified**. `2` is not a lesser `1`: it is the case the whole proposal exists to make diff --git a/prototype/adapters.py b/prototype/adapters.py index eff4971..e165554 100644 --- a/prototype/adapters.py +++ b/prototype/adapters.py @@ -63,7 +63,7 @@ class HistoricalHit: class StoreAdapter(Protocol): - """Minimum fail-closed contract for one required retrieval store.""" + """Complete fail-closed surface exercised by the reference controller.""" name: str required: bool @@ -79,6 +79,34 @@ def quarantine(self, artifact_ids: tuple[str, ...]) -> None: ... def is_quarantined(self, artifact_id: str) -> bool: ... + def quarantine_coverage(self, root: str) -> Coverage: + """Coverage at the durable quarantine checkpoint, before repair.""" + + ... + + def source_artifact(self, artifact_id: str) -> str: + """Stable lineage node represented by one store artifact.""" + + ... + + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + """Store-owned direct inputs used to classify and rebuild an artifact.""" + + ... + + def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: ... + + def rebuild( + self, artifact_id: str, *, inputs: tuple[str, ...], replacement: str | None + ) -> str: ... + + def recall(self, query: str) -> tuple[Hit, ...]: ... + + def snapshot(self) -> dict[str, str]: + """Inspectable state bound into checkpoint and receipt state roots.""" + + ... + def coverage(self, root: str) -> Coverage: ... def dispositions(self, root: str) -> dict[str, str]: ... @@ -121,6 +149,16 @@ def quarantine(self, artifact_ids: tuple[str, ...]) -> None: def is_quarantined(self, artifact_id: str) -> bool: return artifact_id in self._quarantined + def quarantine_coverage(self, root: str) -> Coverage: + descendants = set(self.enumerate(root)) + if not self.lineage_complete(root): + return Coverage.UNKNOWN + if descendants.issubset(self._quarantined): + return Coverage.VERIFIED + if descendants & self._quarantined: + return Coverage.PARTIAL + return Coverage.FAILED + def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: """Remove an artifact from present-tense recall. @@ -167,6 +205,9 @@ def source_artifact(self, artifact_id: str) -> str: return artifact_id + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + return self._ledger.artifact(artifact_id).inputs + def release(self, artifact_id: str) -> None: """Un-gate without repairing. Only a non-conforming strategy does this.""" @@ -278,7 +319,17 @@ def quarantine(self, artifact_ids: tuple[str, ...]) -> None: def is_quarantined(self, artifact_id: str) -> bool: return artifact_id in self._quarantined - def retire(self, entry_id: str) -> None: + def quarantine_coverage(self, root: str) -> Coverage: + descendants = set(self.enumerate(root)) + if not self.lineage_complete(root): + return Coverage.UNKNOWN + if descendants.issubset(self._quarantined): + return Coverage.VERIFIED + if descendants & self._quarantined: + return Coverage.PARTIAL + return Coverage.FAILED + + def retire(self, entry_id: str, *, superseded_at: str | None = None) -> None: self._retired.add(entry_id) self._text.pop(entry_id, None) @@ -307,6 +358,12 @@ def recall(self, query: str, *, threshold: float = 0.2) -> tuple[Hit, ...]: def source_of(self, entry_id: str) -> str: return self._source_of[entry_id] + def source_artifact(self, entry_id: str) -> str: + return self.source_of(entry_id) + + def repair_inputs(self, entry_id: str) -> tuple[str, ...]: + return self._ledger.artifact(self.source_artifact(entry_id)).inputs + def release(self, entry_id: str) -> None: self._quarantined.discard(entry_id) @@ -378,7 +435,18 @@ def quarantine(self, artifact_ids: tuple[str, ...]) -> None: def is_quarantined(self, artifact_id: str) -> bool: return False - def retire(self, artifact_id: str) -> None: + def quarantine_coverage(self, root: str) -> Coverage: + return Coverage.UNKNOWN + + def source_artifact(self, artifact_id: str) -> str: + return artifact_id + + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + return () + + def retire( + self, artifact_id: str, *, superseded_at: str | None = None + ) -> None: return None def rebuild( @@ -389,6 +457,9 @@ def rebuild( def recall(self, query: str) -> tuple[Hit, ...]: return () + def snapshot(self) -> dict[str, str]: + return {} + def coverage(self, root: str) -> Coverage: return Coverage.UNKNOWN diff --git a/prototype/controller.py b/prototype/controller.py index c0afe4e..52d3935 100644 --- a/prototype/controller.py +++ b/prototype/controller.py @@ -176,8 +176,7 @@ def quarantine(self, erratum: Erratum) -> QuarantineCheckpoint: validated = self.observe(erratum) pre_state_root = self.state_root() root = validated.target_root - gated, coverage_limitations = self._quarantine(root) - unknown = set(coverage_limitations) + gated, checkpoint_coverage, coverage_limitations = self._quarantine(root) records = [] for adapter in sorted(self.adapters, key=lambda item: item.name): limitation = coverage_limitations.get(adapter.name) @@ -186,7 +185,7 @@ def quarantine(self, erratum: Erratum) -> QuarantineCheckpoint: name=adapter.name, required=bool(getattr(adapter, "required", True)), artifact_ids=tuple(sorted(gated[adapter.name])), - coverage="unknown" if adapter.name in unknown else "verified", + coverage=checkpoint_coverage[adapter.name].value, limitation=limitation, ) ) @@ -214,7 +213,11 @@ def repair_quarantined( coverage_limitations = { record.name: record.limitation for record in checkpoint.adapters - if record.coverage == "unknown" and record.limitation is not None + if record.limitation is not None + } + checkpoint_coverage = { + record.name: Coverage(record.coverage) + for record in checkpoint.adapters } self.journal.append(JournalEvent(Phase.REBUILD_BEGIN, {"root": root})) @@ -225,7 +228,11 @@ def repair_quarantined( self.journal.append(JournalEvent(Phase.TEST, dict(triad))) receipt = self._attest( - validated, checkpoint.pre_state_root, coverage_limitations, triad + validated, + checkpoint.pre_state_root, + checkpoint_coverage, + coverage_limitations, + triad, ) self.last_sequence = validated.sequence self._applied[root] = validated.sequence @@ -267,9 +274,11 @@ def _validate_checkpoint( raise CheckpointError(f"checkpoint opaque coverage drifted: {name}") continue current_limitation = self._lineage_limitation(adapter, erratum.target_root) - expected_coverage = "unknown" if current_limitation else "verified" + expected_coverage, current_limitation = self._checkpoint_coverage( + adapter, erratum.target_root, current_limitation + ) if ( - record.coverage != expected_coverage + record.coverage != expected_coverage.value or record.limitation != current_limitation ): raise CheckpointError(f"checkpoint adapter coverage drifted: {name}") @@ -300,8 +309,14 @@ def _lineage_limitation(adapter: StoreAdapter, root: str) -> str | None: complete = audit(root) is True if audit is not None else False except Exception: complete = False - if complete: + snapshot = getattr(adapter, "snapshot", None) + if complete and callable(snapshot): return None + if complete: + return ( + f"{adapter.name}: adapter exposes no state snapshot for {root}; " + "checkpoint and receipt state roots cannot bind its mutations" + ) return ( f"{adapter.name}: enumeration returned a result but the adapter did " f"not establish complete root-specific lineage for {root}; empty or " @@ -316,11 +331,41 @@ def _feed_view_limitation() -> str: "non-equivocation without an external witnessed or append-only log" ) - def _quarantine(self, root: str) -> tuple[dict[str, list[str]], dict[str, str]]: + @staticmethod + def _checkpoint_coverage( + adapter: StoreAdapter, root: str, limitation: str | None + ) -> tuple[Coverage, str | None]: + """Read quarantine-phase coverage without inventing success. + + This is deliberately distinct from ``coverage(root)``, which evaluates + final repair dispositions. Missing, raising, or malformed checkpoint + evidence becomes ``unknown`` and is bound into the durable record. + """ + + report = getattr(adapter, "quarantine_coverage", None) + try: + result = report(root) if report is not None else Coverage.UNKNOWN + except Exception: + result = Coverage.UNKNOWN + if not isinstance(result, Coverage): + result = Coverage.UNKNOWN + if report is None or result is Coverage.UNKNOWN: + limitation = limitation or ( + f"{adapter.name}: adapter did not establish quarantine-phase " + f"coverage for {root}" + ) + if limitation is not None and result is Coverage.VERIFIED: + result = Coverage.UNKNOWN + return result, limitation + + def _quarantine( + self, root: str + ) -> tuple[dict[str, list[str]], dict[str, Coverage], dict[str, str]]: """Gate every known descendant everywhere, before any rebuild starts.""" self.journal.append(JournalEvent(Phase.QUARANTINE_BEGIN, {"root": root})) gated: dict[str, list[str]] = {} + checkpoint_coverage: dict[str, Coverage] = {} limitations: dict[str, str] = {} for adapter in self.adapters: try: @@ -331,19 +376,25 @@ def _quarantine(self, root: str) -> tuple[dict[str, list[str]], dict[str, str]]: acknowledge(root) limitations[adapter.name] = self._opaque_limitation(adapter.name) gated[adapter.name] = [] + checkpoint_coverage[adapter.name] = Coverage.UNKNOWN continue lineage_limitation = self._lineage_limitation(adapter, root) - if lineage_limitation is not None: - limitations[adapter.name] = lineage_limitation adapter.quarantine(descendants) gated[adapter.name] = list(descendants) + reported, limitation = self._checkpoint_coverage( + adapter, root, lineage_limitation + ) + checkpoint_coverage[adapter.name] = reported + if limitation is not None: + limitations[adapter.name] = limitation self.journal.append(JournalEvent(Phase.QUARANTINE_COMPLETE, gated)) - return gated, limitations + return gated, checkpoint_coverage, limitations def _attest( self, erratum: Erratum, pre_state_root: str, + checkpoint_coverage: dict[str, Coverage], coverage_limitations: dict[str, str], triad: dict[str, str], ) -> Receipt: @@ -357,13 +408,8 @@ def _attest( if not getattr(adapter, "required", True): continue reported = adapter.coverage(root) - # Missing lineage evidence may downgrade a claimed success, never - # upgrade a failure the adapter already reported. - stores[adapter.name] = ( - Coverage.UNKNOWN - if adapter.name in coverage_limitations - and reported is Coverage.VERIFIED - else reported + stores[adapter.name] = self._conservative_coverage( + checkpoint_coverage[adapter.name], reported ) receipt = Receipt( importer=self.name, @@ -388,6 +434,21 @@ def _attest( ) return replace(receipt, signature=self.signer.sign(receipt.signable())) + @staticmethod + def _conservative_coverage( + checkpoint: Coverage, final: Coverage + ) -> Coverage: + """Final repair cannot erase an earlier coverage failure or limitation.""" + + if Coverage.FAILED in {checkpoint, final}: + return Coverage.FAILED + rank = { + Coverage.VERIFIED: 0, + Coverage.PARTIAL: 1, + Coverage.UNKNOWN: 2, + } + return max((checkpoint, final), key=rank.__getitem__) + # -- probes -------------------------------------------------------- def _run_triad(self, erratum: Erratum) -> dict[str, str]: diff --git a/prototype/sqlite_store.py b/prototype/sqlite_store.py index bae8ced..946496c 100644 --- a/prototype/sqlite_store.py +++ b/prototype/sqlite_store.py @@ -144,6 +144,30 @@ def is_quarantined(self, artifact_id: str) -> bool: ).fetchone() return bool(row and row[0]) + def quarantine_coverage(self, root: str) -> Coverage: + descendants = set(self.enumerate(root)) + if not self.lineage_complete(root): + return Coverage.UNKNOWN + quarantined = { + artifact_id + for artifact_id in descendants + if self.is_quarantined(artifact_id) + } + if quarantined == descendants: + return Coverage.VERIFIED + if quarantined: + return Coverage.PARTIAL + return Coverage.FAILED + + def source_artifact(self, artifact_id: str) -> str: + return artifact_id + + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + row = self._db.execute( + "SELECT inputs FROM artifacts WHERE artifact_id=?", (artifact_id,) + ).fetchone() + return tuple(json.loads(row[0])) if row else () + def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: """Remove the row, and remember what it said. diff --git a/prototype/strategies.py b/prototype/strategies.py index 25575c1..4454679 100644 --- a/prototype/strategies.py +++ b/prototype/strategies.py @@ -52,7 +52,6 @@ def apply(self, importer, erratum: Erratum, gated: dict[str, list[str]]) -> None attempted. Adding a store must not require editing this file. """ - ledger = importer.ledger replacement = erratum.replacement # Only a supersession retains the old value, and only as scoped # history. `valid_from` is the instant it stopped being true. @@ -60,37 +59,32 @@ def apply(self, importer, erratum: Erratum, gated: dict[str, list[str]]) -> None erratum.valid_from if erratum.operation is Operation.SUPERSEDE else None ) - def source_of(adapter, item: str) -> str: - resolve = getattr(adapter, "source_artifact", None) or getattr( - adapter, "source_of", None - ) - return resolve(item) if resolve else item - # Pass one: retire everything that descends directly from the root, so # pass two knows which inputs are no longer valid. retired: set[str] = set() for store, items in gated.items(): adapter = importer.adapter(store) for item in items: - source = source_of(adapter, item) - if source in ledger.artifact_ids() and ledger.artifact(source).inputs: + source = adapter.source_artifact(item) + if adapter.repair_inputs(item): continue - try: - adapter.retire(item, superseded_at=superseded_at) - except TypeError: - adapter.retire(item) + adapter.retire(item, superseded_at=superseded_at) retired.add(source) # Pass two: rebuild the mixed artifacts from what survived. for store, items in gated.items(): adapter = importer.adapter(store) for item in items: - source = source_of(adapter, item) + source = adapter.source_artifact(item) if source in retired: continue adapter.rebuild( item, - inputs=ledger.valid_inputs(source, retired=retired), + inputs=tuple( + value + for value in adapter.repair_inputs(item) + if value not in retired + ), replacement=replacement, ) diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 0fda7a0..ce8f4e5 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -92,9 +92,9 @@ } G2_MATRIX_CURRENT_EVIDENCE = ( "Phase 2 implementation includes conflict-disclosed remediation for split-view " - "equivocation and unsupported empty enumeration, plus schemas, checkpoints, semantic " - "probes, key rotation, invalid-target, confidentiality, and receipt binding; no " - "qualifying independent review is recorded." + "equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract " + "completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, " + "and receipt binding; no qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( "Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation." diff --git a/scripts/claim_guard.py b/scripts/claim_guard.py index be95e5f..7b449d9 100644 --- a/scripts/claim_guard.py +++ b/scripts/claim_guard.py @@ -97,6 +97,16 @@ "An empty enumeration is not evidence of complete lineage.", "empty enumeration is not verified coverage", ), + ( + "IDEA.md", + "A durable checkpoint records adapter-supplied quarantine coverage; final repair cannot upgrade a worse checkpoint result.", + "checkpoint coverage cannot be fabricated or upgraded", + ), + ( + "IDEA.md", + "The published adapter contract must expose every controller and repair operation without hidden reference-ledger dependencies.", + "adapter contract exposes the complete runtime surface", + ), ( "IDEA.md", "A receipt establishes the signed event accepted by one importer.", diff --git a/spec/README.md b/spec/README.md index c03e89e..06cc4ac 100644 --- a/spec/README.md +++ b/spec/README.md @@ -9,6 +9,11 @@ than this one can be built and checked against the same contract. | `receipt.schema.json` | What an importer returns after acting on one. | | `vectors/manifest.json` | Every vector, which schema it belongs to, whether it must validate, and — for an invalid vector — the text that must appear in the rejection. | | `vendor/json-schema-test-suite/` | Cases from the official JSON-Schema-Test-Suite, vendored unmodified. | +| `semantic/probes.json` | Named, strict semantic-probe sets for offline conformance. | +| `semantic/verifier-config.json` | Exact synthetic verifier configuration, whose canonical digest binds every observation. | +| `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | +| `vectors/protocol-manifest.json` | Executable key-rotation, concurrency, invalid-target, and confidentiality cases that JSON Schema cannot express. | +| `vectors/receipt-binding-mutations.json` | Valid-domain mutations proving every signed receipt field is bound. | Receipt conformance evaluates two independent properties. D1 authenticity verifies the importer and every signable byte. D2 coverage truthfulness checks @@ -21,11 +26,12 @@ root-specific lineage-completeness evidence as `unknown`. Receipts also carry a signed limitation that importer-local sequencing cannot establish global owner non-equivocation across split views. The wire coverage vocabulary remains the four terminal results `verified`, `partial`, `unknown`, and `failed`. -| `semantic/probes.json` | Named, strict semantic-probe sets for offline conformance. | -| `semantic/verifier-config.json` | Exact synthetic verifier configuration, whose canonical digest binds every observation. | -| `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | -| `vectors/protocol-manifest.json` | Executable key-rotation, concurrency, invalid-target, and confidentiality cases that JSON Schema cannot express. | -| `vectors/receipt-binding-mutations.json` | Valid-domain mutations proving every signed receipt field is bound. | + +The importer-local checkpoint uses the same terminal vocabulary but measures a +different phase: whether the known affected set is durably gated before repair. +It records adapter-supplied quarantine coverage, and the final signed receipt +cannot improve past a worse checkpoint result. Final `coverage(root)` measures +post-repair dispositions and substrate evidence. ## Offline semantic-probe fixtures diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 1e090cc..608b0b1 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -15,6 +15,7 @@ CannotEnumerate, MarkdownAdapter, OpaqueAdapter, + StoreAdapter, VectorAdapter, ) from prototype.lineage import Artifact, LineageLedger @@ -74,6 +75,26 @@ def test_still_valid_inputs_exclude_the_retired_root(self) -> None: self.assertEqual(retained, ("fact:venue", "fact:budget")) +class StoreAdapterContractIsComplete(unittest.TestCase): + def test_protocol_names_every_method_the_reference_repair_invokes(self) -> None: + required = { + "enumerate", + "lineage_complete", + "quarantine", + "is_quarantined", + "quarantine_coverage", + "source_artifact", + "repair_inputs", + "retire", + "rebuild", + "recall", + "snapshot", + "coverage", + "dispositions", + } + self.assertTrue(required.issubset(vars(StoreAdapter))) + + class MarkdownAdapterExposesExactLineage(unittest.TestCase): def setUp(self) -> None: self.ledger = ledger_with_a_mixed_summary() diff --git a/tests/test_claim_guard.py b/tests/test_claim_guard.py index f738597..1941b18 100644 --- a/tests/test_claim_guard.py +++ b/tests/test_claim_guard.py @@ -117,6 +117,34 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("empty enumeration", result.stdout) + def test_checkpoint_cannot_upgrade_adapter_coverage(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "IDEA.md", + "A durable checkpoint records adapter-supplied quarantine coverage; " + "final repair cannot upgrade a worse checkpoint result.", + "A durable checkpoint may infer verified coverage and final repair may " + "upgrade an earlier result.", + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("checkpoint coverage", result.stdout) + + def test_adapter_contract_cannot_hide_reference_ledger_dependency(self) -> None: + def mutate(root: Path) -> None: + rewrite( + root / "IDEA.md", + "The published adapter contract must expose every controller and " + "repair operation without hidden reference-ledger dependencies.", + "The published adapter contract may omit repair operations and depend " + "on the reference ledger implicitly.", + ) + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("adapter contract", result.stdout) + def test_receipt_cannot_claim_global_feed_consistency(self) -> None: def mutate(root: Path) -> None: rewrite( diff --git a/tests/test_controller.py b/tests/test_controller.py index e2fefc7..bfbd295 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -11,8 +11,9 @@ import unittest from prototype.adapters import Coverage from prototype.checkpoints import CheckpointError, QuarantineCheckpoint -from prototype.controller import Phase -from prototype.errata import Erratum, Operation +from prototype.controller import Importer, Phase +from prototype.errata import Erratum, Operation, RootRegistry +from prototype.lineage import LineageLedger from prototype.scenario import DIET, build_importer from prototype.signing import DemoSigner from prototype.strategies import ( @@ -42,6 +43,29 @@ def quarantine(self, artifact_ids: tuple[str, ...]) -> None: def is_quarantined(self, artifact_id: str) -> bool: return False + def quarantine_coverage(self, root: str) -> Coverage: + return Coverage.VERIFIED + + def source_artifact(self, artifact_id: str) -> str: + return artifact_id + + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + return () + + def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: + return None + + def rebuild( + self, artifact_id: str, *, inputs: tuple[str, ...], replacement: str | None + ) -> str: + return replacement or "" + + def recall(self, query: str): + return () + + def snapshot(self) -> dict[str, str]: + return {} + def coverage(self, root: str) -> Coverage: return Coverage.VERIFIED @@ -63,6 +87,138 @@ def coverage(self, root: str) -> Coverage: return Coverage.FAILED +class PartialQuarantineAdapter(SilentLineageAdapter): + """The adapter can enumerate but reports incomplete quarantine coverage.""" + + name = "partial_quarantine_store" + + def __init__(self) -> None: + self.quarantined: set[str] = set() + + def enumerate(self, root: str) -> tuple[str, ...]: + return ("external:root",) + + def lineage_complete(self, root: str) -> bool: + return True + + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: + self.quarantined.update(artifact_ids) + + def is_quarantined(self, artifact_id: str) -> bool: + return artifact_id in self.quarantined + + def quarantine_coverage(self, root: str) -> Coverage: + return Coverage.PARTIAL + + +class MissingQuarantineCoverageAdapter(PartialQuarantineAdapter): + name = "missing_quarantine_coverage" + quarantine_coverage = None + + +class IndependentStoreAdapter: + """A complete adapter whose lineage lives in its own store, not the ledger.""" + + name = "independent_store" + required = True + + def __init__(self) -> None: + self.records = { + "external:diet": {"content": "is vegetarian", "inputs": ()}, + "external:venue": {"content": "prefers quiet restaurants", "inputs": ()}, + "external:budget": {"content": "moderate budget", "inputs": ()}, + "external:summary": { + "content": "is vegetarian; prefers quiet restaurants; moderate budget", + "inputs": ( + "external:diet", + "external:venue", + "external:budget", + ), + }, + } + self.quarantined: set[str] = set() + self.retired: set[str] = set() + self.rebuilt: set[str] = set() + + def enumerate(self, root: str) -> tuple[str, ...]: + return ("external:diet", "external:summary") + + def lineage_complete(self, root: str) -> bool: + return True + + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: + self.quarantined.update(artifact_ids) + + def is_quarantined(self, artifact_id: str) -> bool: + return artifact_id in self.quarantined + + def quarantine_coverage(self, root: str) -> Coverage: + return ( + Coverage.VERIFIED + if set(self.enumerate(root)).issubset(self.quarantined) + else Coverage.FAILED + ) + + def source_artifact(self, artifact_id: str) -> str: + return artifact_id + + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + return self.records[artifact_id]["inputs"] + + def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: + self.retired.add(artifact_id) + + def rebuild( + self, artifact_id: str, *, inputs: tuple[str, ...], replacement: str | None + ) -> str: + parts = [self.records[item]["content"] for item in inputs] + if replacement: + parts.insert(0, replacement) + content = "; ".join(parts) + self.records[artifact_id]["content"] = content + self.rebuilt.add(artifact_id) + self.quarantined.discard(artifact_id) + return content + + def recall(self, query: str): + from prototype.adapters import Hit + + return tuple( + Hit(artifact_id, record["content"]) + for artifact_id, record in self.records.items() + if artifact_id not in self.quarantined + and artifact_id not in self.retired + and query.lower() in record["content"].lower() + ) + + def snapshot(self) -> dict[str, str]: + return { + artifact_id: record["content"] + for artifact_id, record in self.records.items() + if artifact_id not in self.retired + } + + def coverage(self, root: str) -> Coverage: + return ( + Coverage.VERIFIED + if "external:diet" in self.retired + and "external:summary" in self.rebuilt + else Coverage.FAILED + ) + + def dispositions(self, root: str) -> dict[str, str]: + return { + item: ( + "retired" + if item in self.retired + else "rebuilt" + if item in self.rebuilt + else "quarantined-only" + ) + for item in self.enumerate(root) + } + + def supersede(sequence: int = 1) -> Erratum: return OWNER.sign_erratum( Erratum( @@ -174,6 +330,37 @@ def test_drifted_checkpoint_is_refused_before_rebuild(self) -> None: importer.repair_quarantined(supersede(), drifted) self.assertNotIn(Phase.REBUILD_BEGIN, [event.phase for event in importer.journal]) + def test_checkpoint_preserves_adapter_partial_quarantine_coverage(self) -> None: + importer = build_importer(OWNER, include_opaque=False) + importer.adapters.append(PartialQuarantineAdapter()) + checkpoint = importer.quarantine(supersede()) + record = next( + item for item in checkpoint.adapters + if item.name == "partial_quarantine_store" + ) + self.assertEqual(record.coverage, "partial") + receipt = importer.repair_quarantined(supersede(), checkpoint) + self.assertEqual( + receipt.stores["partial_quarantine_store"], Coverage.PARTIAL + ) + + def test_missing_quarantine_coverage_fails_closed_as_unknown(self) -> None: + importer = build_importer(OWNER, include_opaque=False) + importer.adapters.append(MissingQuarantineCoverageAdapter()) + checkpoint = importer.quarantine(supersede()) + record = next( + item for item in checkpoint.adapters + if item.name == "missing_quarantine_coverage" + ) + self.assertEqual(record.coverage, "unknown") + self.assertIn("quarantine-phase", record.limitation) + + def test_unknown_checkpoint_cannot_be_improved_to_partial(self) -> None: + self.assertEqual( + Importer._conservative_coverage(Coverage.UNKNOWN, Coverage.PARTIAL), + Coverage.UNKNOWN, + ) + class EveryDescendantGetsADisposition(unittest.TestCase): """Acceptance: every known descendant receives an explicit disposition.""" @@ -236,6 +423,21 @@ def test_the_summary_keeps_the_two_retained_facts(self) -> None: self.assertIn("moderate budget", summary) self.assertIn("eats meat again", summary) + def test_an_independent_adapter_needs_no_reference_ledger_registration(self) -> None: + adapter = IndependentStoreAdapter() + importer = Importer( + "independent-importer", + ledger=LineageLedger(), + adapters=[adapter], + signer=DemoSigner(b"independent-importer-secret"), + owner=OWNER.public, + roots=RootRegistry({DIET}), + ) + receipt = importer.repair(supersede()) + self.assertEqual(receipt.aggregate, Coverage.VERIFIED) + self.assertIn("eats meat again", adapter.records["external:summary"]["content"]) + self.assertIn("quiet restaurants", adapter.records["external:summary"]["content"]) + class OperationsStayDistinct(unittest.TestCase): def test_correction_does_not_preserve_the_false_history(self) -> None: diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 620a2b4..011021e 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -435,7 +435,7 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation and unsupported empty enumeration, plus schemas, checkpoints, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded.", + "Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded.", "local tests prove readiness.", ) From c3e02a2e6dea36e92836aac7dd0c3781158877b8 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 17:10:48 +0200 Subject: [PATCH 59/88] docs(publication): record current feedback target --- docs/PUBLICATION_LOG.md | 46 ++++++++++++++++++++------------ publication/active-surfaces.json | 36 +++++++------------------ scripts/check_publication.py | 18 ++++--------- tests/test_publication.py | 10 +++---- 4 files changed, 48 insertions(+), 62 deletions(-) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 51796f7..7e1acea 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -24,7 +24,11 @@ readiness gate. Last verified: 2026-08-13. | GitHub Discussions announcement | [Discussion #9](https://github.com/thomaswillner/llm-errata/discussions/9), `LLM Errata needs falsifiers, independent reviewers, adapters, and real memory systems` | Created 2026-08-12 after Discussions was enabled (`has_discussions=true`). It links the repository, public issues, and PR #8; states Phase 2 incomplete, G2 `BLOCKED`, and `NOT_PROD_READY`. Publication/invitation only: no response yet and not external evidence. | | Historical Phase 2 review target | Commit [`50e895fbfec544b16c94caa07bf2d1f4049a42e2`](https://github.com/thomaswillner/llm-errata/commit/50e895fbfec544b16c94caa07bf2d1f4049a42e2), digest `9547aec8328b601489dda067c6e62f287229b2b24a413dac2c9e7be98e429804` | Stale: this target predated the accepted prior-art corrections and must not be used for a current review. Preserved as historical publication evidence. | | Corrected complete Phase 2 review target | Commit [`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878), digest `03abc492319b875a7d528e0e8de05714bc5a7219b42031fc7c3f42cff1f0bf14` | Complete corrected specification, prior-art, licensing, and cryptographic-qualification surface. Checkout and committed-source digests match; exact-target CI passed Python 3.11 and 3.13. Internal evidence does not satisfy G2. | -| Current feedback-remediation target | Commit [`a477fe4f5c86730031b6285d9505778fb8eec060`](https://github.com/thomaswillner/llm-errata/commit/a477fe4f5c86730031b6285d9505778fb8eec060), digest `a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3` | Remediates same-importer conflict persistence, signed split-view limitations, and unsupported empty enumeration. Checkout and committed-tree digests match; exact-target [CI run 31698542878](https://github.com/thomaswillner/llm-errata/actions/runs/31698542878) passed Python 3.11 and 3.13. Internal evidence does not satisfy G2. | +| Former feedback-remediation target | Commit [`a477fe4f5c86730031b6285d9505778fb8eec060`](https://github.com/thomaswillner/llm-errata/commit/a477fe4f5c86730031b6285d9505778fb8eec060), digest `a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3` | Remediated same-importer conflict persistence, signed split-view limitations, and unsupported empty enumeration. Later adapter testing found checkpoint-coverage and contract defects, so this target is historical. Exact-target [CI run 31698542878](https://github.com/thomaswillner/llm-errata/actions/runs/31698542878) passed Python 3.11 and 3.13. | +| Current adapter-contract remediation target | Commit [`ac4468faf73c2cc7949dd29b2a2a151f5bd23116`](https://github.com/thomaswillner/llm-errata/commit/ac4468faf73c2cc7949dd29b2a2a151f5bd23116), digest `7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12` | Preserves adapter-supplied checkpoint coverage, exposes complete adapter runtime surface, and removes hidden reference-ledger coupling. Checkout and committed-tree digests match; exact-target [CI run 31713580146](https://github.com/thomaswillner/llm-errata/actions/runs/31713580146) passed Python 3.11 and 3.13. Internal evidence does not satisfy G2 or G4. | +| Adapter checkpoint finding | [Inspeximus implementation report](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5280326386) | Reproduced `partial` adapter quarantine coverage being recorded as `verified`; accepted and fixed. Interested-party technical evidence, not a qualifying G2 review. | +| Adapter provenance and contract findings | [Inspeximus v2.7.0 report](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5281287746) | Preserves v2.6.1 contamination disclosure, claims a clean-room v2.7.0 rewrite, and reports omitted repair methods plus hidden ledger coupling. Findings reproduced and fixed; independence and current-target behavior remain unverified. | +| Current human remediation reply | [Point-by-point response](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5282207719) | Read back from GitHub with author and body verified. Publishes commit `ac4468f`, digest `7e0d6c88…`, exact-head CI, accepted fixes, provenance boundary, current vector set, and request to rebind Inspeximus. Recruitment and response only; G2 and G4 remain `BLOCKED`. | | Interested-party technical review | [Two counterexamples](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5276848751) | Bound to the corrected target with explicit Inspeximus/market conflict. Reproduced same-importer versus split-view equivocation limits and unsupported empty-enumeration success. Useful external technical evidence, but not a qualifying complete independent G2 review. | | Cross-project remediation and adapter offer | [Inspeximus follow-up](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5279081884) | Inspeximus source commit `3661102` fixes an adjacent known-hole audit defect and offers an independently authored adapter plus benchmark harness with explicit commercial interest. No adapter artifact exists yet; the producer cannot also validate its own adapter for G4. | | Cryptography maintainer response | [Pure-Python Ed25519 reference](https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5275036522) | Useful referral and evidence that the outreach lacked plain-language context. The linked pure-Python project is a comparison oracle, not audited constant-time G3 evidence. | @@ -43,22 +47,28 @@ readiness gate. Last verified: 2026-08-13. | Phase 2 completion announcement | [Discussion #9 completion update](https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17993648) | Announces internal completion and routes falsifiers to Issues #4, #5, #6, and #10. Publication only; it records no external acceptance or readiness-gate change. | | Six-step readiness checkpoint | [PR #8 checkpoint](https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5272526975) | Audits every approved production-readiness step against exact internal and external evidence. Records `NOT_PROD_READY`, the external blockers, PR #3 status, validation, and publication boundaries without claiming certification. | -One current-target invitation now asks the conflict-disclosed Inspeximus -maintainer for an independently authored adapter and harness. It does not -satisfy G2 or G4 by itself. Prior role-specific calls remain historical and -visible, but their target is superseded by the current PR checkpoint. No -produced adapter, separate validator, operated-system approval, or CODEOWNER +One current-target reply asks the conflict-disclosed Inspeximus maintainer to +rebind the externally authored adapter candidate to the corrected immutable +surface. It does not establish clean-room independence or satisfy G2 or G4 by +itself. Prior role-specific calls remain historical and visible. No qualifying +second adapter, separate validator, operated-system approval, or CODEOWNER volunteer is recorded. The current complete conformance review target is source commit -[`a477fe4f5c86730031b6285d9505778fb8eec060`](https://github.com/thomaswillner/llm-errata/commit/a477fe4f5c86730031b6285d9505778fb8eec060) +[`ac4468faf73c2cc7949dd29b2a2a151f5bd23116`](https://github.com/thomaswillner/llm-errata/commit/ac4468faf73c2cc7949dd29b2a2a151f5bd23116) with canonical Phase 2 surface digest -`a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3`. +`7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12`. The digest was recomputed from checkout and committed tree with an exact match. The exact-target [GitHub Actions run -31698542878](https://github.com/thomaswillner/llm-errata/actions/runs/31698542878) +31713580146](https://github.com/thomaswillner/llm-errata/actions/runs/31713580146) passed Python 3.11 and Python 3.13. +The former feedback-remediation target is source commit +[`a477fe4f5c86730031b6285d9505778fb8eec060`](https://github.com/thomaswillner/llm-errata/commit/a477fe4f5c86730031b6285d9505778fb8eec060) +with digest `a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3`. +It is historical because the later adapter implementation found checkpoint- +coverage and adapter-contract defects in that surface. + The former complete conformance review target is source commit [`08b95263c9ed700c43aea0b285696956cc23e878`](https://github.com/thomaswillner/llm-errata/commit/08b95263c9ed700c43aea0b285696956cc23e878) with canonical Phase 2 surface digest @@ -126,11 +136,13 @@ interoperability evidence. ## Readiness boundary -GitHub calls, invitations, CI results, maintainer work, and conflict-disclosed -interested-party review are not qualifying independent evidence. Phase 2 -implementation now includes remediation for same-view versus split-view -equivocation and unsupported empty enumeration. G2 remains `BLOCKED` pending a -dated qualifying independent review of commit -`a477fe4f5c86730031b6285d9505778fb8eec060` and digest -`a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3`. -Repository verdict remains `NOT_PROD_READY`. +GitHub calls, replies, CI results, maintainer work, and conflict-disclosed +interested-party review are not qualifying independent evidence. Phase 2 now +also preserves adapter-supplied quarantine coverage, exposes the complete +runtime adapter contract, and removes hidden reference-ledger coupling. G2 +remains `BLOCKED` pending a dated qualifying independent review of commit +`ac4468faf73c2cc7949dd29b2a2a151f5bd23116` and digest +`7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12`. +G4 remains `BLOCKED` pending two independently established current-target +adapters and a separately produced validator. Repository verdict remains +`NOT_PROD_READY`. diff --git a/publication/active-surfaces.json b/publication/active-surfaces.json index d47b015..6202af4 100644 --- a/publication/active-surfaces.json +++ b/publication/active-surfaces.json @@ -1,8 +1,8 @@ { "schema_version": 1, "review_target": { - "commit": "a477fe4f5c86730031b6285d9505778fb8eec060", - "surface_digest": "a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3" + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "surface_digest": "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" }, "license": { "specification_implementation": "irrevocable worldwide royalty-free commercial and non-commercial", @@ -16,38 +16,20 @@ }, "surfaces": [ { - "id": "g4-inspeximus-adapter-call", + "id": "g4-inspeximus-current-target-reply", "kind": "issue-comment", - "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5280210050", + "url": "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5282207719", "published": "2026-08-13", - "commit": "a477fe4f5c86730031b6285d9505778fb8eec060", - "surface_digest": "a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3", - "gates": ["G4"], + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "surface_digest": "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12", + "gates": ["G2", "G4"], "roles": ["inspeximus-adapter-author"], "mentions": ["DanceNitra"], "supersedes": [ - "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5274369945" + "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5280210050", + "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5280225709" ], "evidence_boundary": "recruitment-only" - }, - { - "id": "pr8-feedback-remediation", - "kind": "pull-request-comment", - "url": "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5280225709", - "published": "2026-08-13", - "commit": "a477fe4f5c86730031b6285d9505778fb8eec060", - "surface_digest": "a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3", - "gates": ["G2", "G3", "G4", "G5", "G6"], - "roles": [], - "mentions": [], - "supersedes": [ - "https://github.com/thomaswillner/llm-errata/issues/4#issuecomment-5274367774", - "https://github.com/thomaswillner/llm-errata/issues/5#issuecomment-5274369945", - "https://github.com/thomaswillner/llm-errata/issues/6#issuecomment-5274371747", - "https://github.com/thomaswillner/llm-errata/pull/8#issuecomment-5274373337", - "https://github.com/thomaswillner/llm-errata/discussions/9#discussioncomment-17995307" - ], - "evidence_boundary": "publication-only" } ] } diff --git a/scripts/check_publication.py b/scripts/check_publication.py index 1292477..f062916 100644 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -14,30 +14,22 @@ ROOT = Path(__file__).resolve().parents[1] MANIFEST = ROOT / "publication" / "active-surfaces.json" -CANONICAL_COMMIT = "a477fe4f5c86730031b6285d9505778fb8eec060" +CANONICAL_COMMIT = "ac4468faf73c2cc7949dd29b2a2a151f5bd23116" CANONICAL_DIGEST = ( - "a6908d21a3fbfc71c11da85ff72634a3917205a06d0ec6c5e3f949756c04e3a3" + "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" ) REPOSITORY_URL = "https://github.com/thomaswillner/llm-errata" ALLOWED_GATES = {"G2", "G3", "G4", "G5", "G6"} REQUIRED_ATTRIBUTION = {"LLM Errata", "Thomas Willner", REPOSITORY_URL} REQUIRED_SURFACES = { - "g4-inspeximus-adapter-call": { + "g4-inspeximus-current-target-reply": { "kind": "issue-comment", - "url": f"{REPOSITORY_URL}/issues/4#issuecomment-5280210050", - "gates": ["G4"], + "url": f"{REPOSITORY_URL}/issues/4#issuecomment-5282207719", + "gates": ["G2", "G4"], "roles": ["inspeximus-adapter-author"], "mentions": ["DanceNitra"], "evidence_boundary": "recruitment-only", }, - "pr8-feedback-remediation": { - "kind": "pull-request-comment", - "url": f"{REPOSITORY_URL}/pull/8#issuecomment-5280225709", - "gates": ["G2", "G3", "G4", "G5", "G6"], - "roles": [], - "mentions": [], - "evidence_boundary": "publication-only", - }, } ROOT_KEYS = {"schema_version", "review_target", "license", "surfaces"} diff --git a/tests/test_publication.py b/tests/test_publication.py index de7e8d1..267d4a5 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -90,23 +90,23 @@ def test_missing_required_surface_is_rejected(self) -> None: def test_duplicate_surface_url_is_rejected(self) -> None: def mutate(payload: dict[str, object]) -> None: - payload["surfaces"][1]["url"] = payload["surfaces"][0]["url"] + payload["surfaces"].append(dict(payload["surfaces"][0])) self._assert_manifest_mutation_is_rejected(mutate, "unique surface URLs") def test_duplicate_evidence_role_is_rejected(self) -> None: def mutate(payload: dict[str, object]) -> None: - payload["surfaces"][1]["roles"] = [ + payload["surfaces"][0]["roles"].append( payload["surfaces"][0]["roles"][0] - ] + ) self._assert_manifest_mutation_is_rejected(mutate, "unique evidence roles") def test_duplicate_mention_is_rejected(self) -> None: def mutate(payload: dict[str, object]) -> None: - payload["surfaces"][1]["mentions"] = [ + payload["surfaces"][0]["mentions"].append( payload["surfaces"][0]["mentions"][0] - ] + ) self._assert_manifest_mutation_is_rejected(mutate, "unique GitHub mentions") From c205fa07e818c682f2715e58c02b7e35bfe2ceb0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 21:17:25 +0200 Subject: [PATCH 60/88] docs(publication): record no-login outreach mirrors --- docs/PUBLICATION_LOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 7e1acea..9e85ba8 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -129,6 +129,21 @@ role that overlaps validator authorship. They also do not satisfy G2. | DEV Community | Attempted; authentication-blocked. No verified authenticated editor was available. | No public post URL recorded. | | Medium | Attempted; authentication-blocked. No verified authenticated editor was available. | No public post URL recorded. | | X | Attempted; authentication-blocked. Live signed-out state observed. | No public post URL recorded. | +| publishto.us | [Public challenge page](https://publishto.us/p/oJ88e94KEN) published and read back on 2026-08-13 at 19:07 UTC, then updated and read back after current-target feedback at 19:13 UTC. No account or login was used. | Public recruitment mirror only. It links commit `ac4468faf73c2cc7949dd29b2a2a151f5bd23116`, digest `7e0d6c88…`, repository terms, and `NOT_PROD_READY`. It does not establish review, conformance, interoperability, endorsement, or any G2–G6 pass. | +| Telegra.ph | [Attributed public challenge page](https://telegra.ph/LLM-Errata-please-try-to-break-this-experimental-AI-memory-correction-protocol-08-13) published and read back on 2026-08-13 at 19:08 UTC, then updated and read back after current-target feedback at 19:13 UTC. The official API created a page without user login; author is Thomas Willner and links to the repository. | Public recruitment mirror only. Same immutable target, attribution, `NOT_PROD_READY`, and non-endorsement boundary; no readiness evidence. | +| InstantPost | [Public-stream challenge page](https://instantpost.us/p/di32CWRcqx) published and read back on 2026-08-13 at 19:10 UTC, then updated and read back after current-target feedback at 19:13 UTC. It was visible in the service's public `All posts` stream; no user account or login was used. | Public recruitment mirror only. Same immutable target and boundary; publication and public-stream visibility do not establish independent evidence or upgrade readiness. | + +All three successful no-login mirrors use the same reviewed publication copy; +the final amended copy SHA-256 is +`89cdc0dea3bae126f12250a40f84cb385f36b7f8f8a8bd875bab22cd86f21365`. It identifies Thomas Rainer Willner as the +responsible publishing author, discloses substantial AI assistance, names the +commercial and non-commercial attribution requirement, and records the latest +interested-party result as 28/28 published wire/controller cases with only 1/28 +reaching an adapter. The producer explicitly states that result does not +validate adapter behavior or move G2 or G4. Write.as returned HTTP/application +code `201` with `id=contentisblocked` and no public URL; InstantPost's first +unauthenticated REST attempt returned `401` before its documented no-account +API-key generation path was used. Neither failed attempt is a public placement. Zenodo and OSF remain deferred until a stable reviewed release. arXiv and standards engagement remain deferred pending a paper-quality manuscript and From 481d329e1763b658e6cd355bea3e5956c0fa1df3 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 22:48:43 +0200 Subject: [PATCH 61/88] docs: design adapter conformance hardening --- ...-conformance-validator-hardening-design.md | 232 ++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md diff --git a/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md new file mode 100644 index 0000000..726b46a --- /dev/null +++ b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md @@ -0,0 +1,232 @@ +# Adapter Conformance and Validator Hardening Design + +**Date:** 2026-08-13 +**Owner:** Thomas Willner +**Status:** Approved for implementation +**Target release:** 0.4.0 + +## Purpose + +LLM Errata currently exercises its wire contract, controller, semantic +aggregator, and receipt binding more deeply than it exercises independently +authored store adapters. Rastislav Drahos reported that an external runner +could reproduce 28 of 28 published cases while only one case reached any store +adapter. He then published candidate adapter cases and validator anti-vacuity +requirements at immutable commit +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. + +This change adopts the verified behavioral findings without copying or +vendoring the contributed runner. It adds an independently authored, +provider-neutral adapter-conformance corpus and a fail-closed validator whose +own controls must demonstrably fail against flattering implementations. + +Release 0.4.0 remains an experimental conformance proposal and tested +reference implementation. It does not change the project verdict from +`NOT_PROD_READY`, and it does not satisfy G2 or G4. + +## Ownership, licence, and provenance + +- LLM Errata, its repository, specification, project name, and files authored + in this repository remain owned and governed by Thomas Willner. +- Repository publication and versioned releases do not transfer copyright. +- Rastislav Drahos and DanceNitra retain copyright in their candidate fixture. + Its MIT licence permits reuse but does not assign ownership. +- This repository will not copy or vendor that runner or its fixture files. + Implementation is independently authored from the accepted behavioral + requirements and the LLM Errata normative contract. +- Documentation must credit Rastislav Drahos and DanceNitra for the reported + adapter-coverage gap, duplicate-preservation counterexample, candidate cases, + and anti-vacuity findings. It must identify the immutable source commit, + MIT licence, interested-party conflict, and disclosed Claude Opus 5 + co-authorship. +- Existing LLM Errata licence terms remain unchanged. Any product or + commercial implementation remains subject to repository terms, including + accessible credit to LLM Errata and Thomas Willner and any required written + grant. Technical adoption, contributor credit, or release publication does + not imply endorsement, partnership, certification, or commercial permission. + +## Normative preservation rule + +Preservation is strengthened from boolean recallability to bounded +multiplicity preservation. + +Within a declared, inspectable adapter scope, repair must not increase the +active multiplicity of a preserved proposition unless the erratum explicitly +requires an additional assertion. A proposition is identified by the +adapter-provided stable proposition identity used by its conformance +observation interface, not by lossy text normalization or embedding +similarity. + +The conformance observation for each active proposition contains: + +- a stable, provider-local proposition identifier; +- a content-free or synthetic fixture label suitable for comparison; and +- an active assertion count. + +Adapters unable to expose proposition identity or active multiplicity cannot +pass the cardinality case. They report `unknown` for that observation. This +rule is scoped evidence: it does not claim semantic uniqueness outside the +declared adapter surface. + +## Corpus format + +`spec/adapter-conformance.json` is a versioned, provider-neutral corpus. Each +case contains: + +- stable case ID and operation; +- synthetic initial state; +- complete expected checkpoint, aggregate, triad, receipt, and store + observations; +- required adapter method calls; +- a pinned normative source path and exact quotation; +- one named mutation and the complete result that mutation must produce; and +- provenance and evidence-boundary fields. + +Partial expected outcomes are forbidden. A case fails when any extra triad +failure, worse aggregate result, missing store observation, unexpected +exception, or unspecified result appears. + +Initial behavioral cases cover: + +1. undeclared derivatives cannot become `verified`; +2. complete lineage can reach `verified`; +3. a rebuild cannot increase preserved-proposition multiplicity; +4. erasure evidence is content-free and non-vacuous; and +5. collateral survives supersession while the complete repair still passes. + +## Validator architecture + +`prototype/conformance.py` owns parsing, source binding, execution, and +result comparison. It exposes one public entry point: + +`validate_adapter_conformance(corpus_path, source_root, binding_factory) -> ConformanceReport`. + +The validator is standard-library only. + +### Source identity + +The corpus binds: + +- exact LLM Errata Git commit; +- canonical conformance-surface digest; +- fixture schema version; and +- every normative source path and exact quotation. + +Validation refuses a dirty or different source tree, a commit mismatch, a +digest mismatch, a missing source, or quotation drift. Source mismatch is an +invalid run, never a case failure or pass. + +### Target-instance tracing + +Required calls are observed through a proxy around the exact adapter instance +given to the controller. The proxy records protocol member access and calls. +Global frame names, unrelated modules, helper functions, and another object +with the same method name cannot satisfy a positive control. + +### Mutation controls + +Every case runs twice: + +1. honest binding; +2. the declared flattering mutation installed through the public adapter or + validator seam. + +The mutation must complete and produce the exact declared counter-result. +Any unexpected exception, missing result, different failure, or unchanged pass +fails the control. Exceptions count only when the corpus explicitly declares +that exact exception type and message as the required outcome. + +### Validator anti-vacuity controls + +Three executable meta-tests attack the validator: + +- an empty receipt cannot satisfy confidentiality; +- no-op feed verification cannot satisfy accept-side feed cases; and +- a constant-`unknown` aggregator cannot satisfy the semantic corpus. + +These are validator requirements, not adapter scores. A release cannot claim +validator readiness unless all three mutations are rejected for their +specified semantic reason. + +## Reference binding + +The LLM Errata reference binding uses only public adapter/controller +interfaces plus a conformance observation seam. Production adapters are not +required to expose raw private store contents. A binding may expose only +synthetic fixture identities and counts for an isolated conformance run. + +The corpus and validator must run against the repository reference adapter. +Passing the reference binding demonstrates harness behavior, not independent +interoperability. + +## CLI and evidence + +The existing CLI gains an `adapter-conformance` command accepting: + +- corpus path; +- source root; and +- an importable binding factory. + +Output is one canonical JSON report with source identity, per-case results, +positive controls, mutation controls, anti-vacuity results, provenance, and an +explicit statement that the report is not G2 or G4 evidence. + +Exit codes: + +- `0`: corpus valid and every case/control passes; +- `1`: one or more behavioral or mutation controls fail; +- `2`: source, corpus, binding, or execution evidence is invalid or + inconclusive. + +## Release integration + +Version 0.4.0 includes the current `agent/g2-publication` surface plus this +validator hardening. The release: + +- updates `VERSION`, `CITATION.cff`, README maturity text, supported + security version, readiness ledger/matrix, and changelog; +- records contributor provenance and the independently authored implementation + boundary; +- keeps every G2 through G6 status unchanged unless its existing evidence + contract independently passes; +- backfills the missing GitHub release object for immutable tag `v0.3.0` + without moving that tag; and +- creates immutable tag and GitHub release `v0.4.0` only after exact-head + local checks and GitHub CI pass. + +## Verification seams + +TDD is performed at these public seams: + +1. corpus validation through `validate_adapter_conformance`; +2. target adapter calls through the tracing proxy; +3. exact mutation-result comparison; +4. executable validator anti-vacuity suite; +5. CLI canonical report and exit codes; +6. repository version/readiness/publication validation; and +7. release metadata consistency. + +Required final checks: + +- focused conformance and CLI tests; +- `make check`; +- `make publication`; +- `make links` when source citations change; +- two-axis standards/spec review against this design; +- exact-head CI on Python 3.11 and 3.13; +- immutable tag and GitHub release verification; and +- repository-wide GitHub freshness receipt after the last mutation. + +## Safe stop + +Stop without releasing when any of these occurs: + +- the normative cardinality rule cannot be expressed without provider-specific + private data; +- the validator cannot distinguish its own flattering mutations; +- source identity cannot be bound mechanically; +- current branch integration drops or rewrites existing evidence; +- local or exact-head CI fails; +- a new actionable contributor comment or review changes the required design; + or +- version, tag, release, licence, attribution, or readiness metadata disagree. From 333ffcc98c0b8bd5521906357e7454bd61a0d2f0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 22:55:28 +0200 Subject: [PATCH 62/88] docs: align ownership design with implementation grant --- ...apter-conformance-validator-hardening-design.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md index 726b46a..18a45cd 100644 --- a/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md +++ b/docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md @@ -39,11 +39,15 @@ reference implementation. It does not change the project verdict from and anti-vacuity findings. It must identify the immutable source commit, MIT licence, interested-party conflict, and disclosed Claude Opus 5 co-authorship. -- Existing LLM Errata licence terms remain unchanged. Any product or - commercial implementation remains subject to repository terms, including - accessible credit to LLM Errata and Thomas Willner and any required written - grant. Technical adoption, contributor credit, or release publication does - not imply endorsement, partnership, certification, or commercial permission. +- Existing LLM Errata licence terms remain unchanged. Independent commercial + and non-commercial implementations of the Specification Materials are + already permitted by the repository's irrevocable, royalty-free + implementation grant, subject to accessible credit to LLM Errata and Thomas + Willner and the no-false-endorsement condition. Commercial use, copying, + redistribution, hosting, or adaptation of Reference Code still requires + separate written permission unless another licence expressly applies. + Technical adoption, contributor credit, or release publication does not + imply endorsement, partnership, certification, or transfer of copyright. ## Normative preservation rule From 93e59a12e69273d9c27ad4eecbabdc26325428e9 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 22:57:43 +0200 Subject: [PATCH 63/88] docs: plan adapter conformance release --- ...adapter-conformance-validator-hardening.md | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md diff --git a/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md b/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md new file mode 100644 index 0000000..1e510f9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-adapter-conformance-validator-hardening.md @@ -0,0 +1,231 @@ +# Adapter Conformance and Validator Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish an independently authored adapter-conformance corpus and fail-closed validator as LLM Errata 0.4.0 without upgrading blocked production-readiness gates. + +**Architecture:** A versioned JSON corpus describes complete adapter outcomes, normative predecessor citations, required target-instance calls, and exact mutation counter-results. `prototype/conformance.py` validates and executes the corpus through a provider-neutral binding protocol and reference binding; the CLI emits a canonical report. Release metadata records ownership, contributor provenance, and unchanged readiness status. + +**Tech Stack:** Python 3.11/3.13 standard library, JSON, unittest, existing LLM Errata controller/adapters, GitHub Actions and Releases. + +**Spec:** `docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md` + +## Global Constraints + +- LLM Errata files authored here remain owned by Thomas Willner. +- Do not copy or vendor `DanceNitra/agora` candidate runner or fixture. +- Preserve specific attribution to Rastislav Drahos/DanceNitra and immutable source commit. +- Preserve existing attributed commercial specification-implementation grant and restricted Reference Code boundary. +- Keep G2 through G6 blocked unless their existing independent evidence contracts pass. +- Standard-library only; all fixtures use synthetic data. +- Every production-code behavior starts with a failing public-seam test. +- The corpus pins the predecessor normative target `ac4468faf73c2cc7949dd29b2a2a151f5bd23116` and digest `7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12`. Runtime source identity is separately reported because a file cannot non-circularly contain the commit and digest that include its own bytes. + +--- + +### Task 1: Corpus and source/citation validation + +**Files:** +- Create: `spec/adapter-conformance.json` +- Create: `prototype/conformance.py` +- Create: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `load_corpus(path: Path, source_root: Path) -> AdapterCorpus` +- Produces: `ConformanceInputError` + +- [ ] **Step 1: Write failing corpus validation tests** + +Tests require exact schema keys, immutable commit availability, predecessor surface digest, exact source path/quotation, complete expectations, exact mutation counter-results, and provenance fields. Negative copies change one field at a time and must raise `ConformanceInputError`. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.CorpusValidation -v` +Expected: import failure because `prototype.conformance` does not exist. + +- [ ] **Step 3: Implement typed corpus parser and source binder** + +Use frozen dataclasses, exact-key validation, `git cat-file`/`git show`, and the pinned predecessor manifest/digest algorithm. Reject dirty identity ambiguity only when a run claims current committed identity; normative citation validation always reads immutable target bytes. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance.CorpusValidation -v` +Expected: all corpus validation tests pass. + +- [ ] **Step 5: Commit** + +`git commit -m "feat: define adapter conformance corpus"` + +### Task 2: Target-instance tracing and complete outcome comparison + +**Files:** +- Modify: `prototype/conformance.py` +- Modify: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `TracingAdapter` +- Produces: `compare_complete_outcome(expected, observed) -> tuple[str, ...]` +- Produces: `ConformanceBinding` protocol and `ReferenceConformanceBinding` + +- [ ] **Step 1: Write failing tracing and comparison tests** + +Tests prove an unrelated object with matching method names cannot satisfy the control, every unspecified triad/aggregate failure is rejected, and missing/extra observation fields fail. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.TargetTracing tests.test_conformance.CompleteComparison -v` +Expected: missing types/functions. + +- [ ] **Step 3: Implement proxy, binding protocol, reference binding, and strict comparator** + +Proxy only records calls routed through the exact wrapped instance. Reference binding builds synthetic lineage and exposes stable synthetic proposition labels/counts without private production data. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance.TargetTracing tests.test_conformance.CompleteComparison -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: trace exact adapter conformance outcomes"` + +### Task 3: Exact mutations and validator anti-vacuity controls + +**Files:** +- Modify: `prototype/conformance.py` +- Modify: `tests/test_conformance.py` + +**Interfaces:** +- Produces: `validate_adapter_conformance(corpus_path, source_root, binding_factory) -> ConformanceReport` +- Produces: `run_validator_anti_vacuity_controls() -> tuple[ControlResult, ...]` + +- [ ] **Step 1: Write failing behavioral and mutation tests** + +Tests cover five honest cases, exact counter-results, exception-as-failure, mutation no-op rejection, empty receipt, no-op feed acceptance, and constant-`unknown` semantic aggregation. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `python3 -m unittest tests.test_conformance.AdapterCases tests.test_conformance.AntiVacuity -v` + +- [ ] **Step 3: Implement execution, exact mutations, report, and controls** + +Each case runs honest and mutated bindings independently. Mutation success requires the declared complete semantic counter-result. Unexpected exceptions produce failed control evidence. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run: `python3 -m unittest tests.test_conformance -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: reject vacuous conformance passes"` + +### Task 4: CLI and documentation + +**Files:** +- Modify: `prototype/cli.py` +- Modify: `tests/test_cli.py` +- Modify: `spec/README.md` +- Modify: `prototype/README.md` +- Modify: `IDEA.md` +- Modify: `ROADMAP.md` +- Modify: `THREAT_MODEL.md` +- Modify: `INDEPENDENT_IMPLEMENTATION.md` +- Modify: `SOURCES.md` + +**Interfaces:** +- Produces: `errata adapter-conformance --corpus ... --source-root ... --binding ...` +- Exit `0` pass, `1` behavioral/control failure, `2` invalid/inconclusive evidence. + +- [ ] **Step 1: Write failing subprocess CLI tests** + +Tests assert canonical JSON, source/runtime identity, five cases, three controls, provenance, evidence boundary, and all three exit classes. + +- [ ] **Step 2: Run CLI tests and verify RED** + +Run: `python3 -m unittest tests.test_cli.AdapterConformanceCommand -v` + +- [ ] **Step 3: Implement CLI and document normative multiplicity/provenance boundary** + +CLI imports `module:factory` only when explicitly supplied; default uses reference binding. Documentation states stable proposition identity/count scope and `unknown` behavior. + +- [ ] **Step 4: Run focused and full conformance tests** + +Run: `python3 -m unittest tests.test_conformance tests.test_cli -v` + +- [ ] **Step 5: Commit** + +`git commit -m "feat: publish adapter conformance command"` + +### Task 5: Version 0.4.0 and release/readiness metadata + +**Files:** +- Modify: `VERSION` +- Modify: `CITATION.cff` +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `SECURITY.md` +- Modify: `readiness/production-readiness.json` +- Modify: `PRODUCTION_READINESS.md` +- Modify: `scripts/validate_repo.py` +- Modify: `tests/test_validate_repo.py` +- Modify: `scripts/check_readiness.py` +- Modify: `tests/test_readiness.py` +- Modify: `docs/PUBLICATION_LOG.md` + +**Interfaces:** +- Produces: internally aligned release version `0.4.0` +- Preserves: verdict `NOT_PROD_READY`, G1 `PASS`, G2-G6 `BLOCKED` + +- [ ] **Step 1: Write failing release-alignment tests** + +Tests require 0.4.0 across version, citation, maturity, security, ledger, matrix, changelog, and contributor provenance. They require unchanged gate statuses and current G2 evidence to name adapter conformance without treating it as external evidence. + +- [ ] **Step 2: Run release tests and verify RED** + +Run: `python3 -m unittest tests.test_validate_repo tests.test_readiness -v` + +- [ ] **Step 3: Update metadata and validators** + +Use release date 2026-08-13. Record independent implementation boundary and exact contributor provenance. Do not change licence grants or gate statuses. + +- [ ] **Step 4: Run release tests and verify GREEN** + +Run: `python3 -m unittest tests.test_validate_repo tests.test_readiness -v` + +- [ ] **Step 5: Commit** + +`git commit -m "release: prepare 0.4.0 adapter conformance hardening"` + +### Task 6: Full verification, review, integration, and publication + +**Files:** +- Review all changes since `c205fa07e818c682f2715e58c02b7e35bfe2ceb0` + +- [ ] **Step 1: Run full local gates** + +Run: `make check` +Run: `make publication` +Run: `make links` because `SOURCES.md` changes. + +- [ ] **Step 2: Run two-axis standards/spec review** + +Compare `c205fa0...HEAD` against AGENTS.md, CONTRIBUTING.md, and approved design. Fix all Critical/Important findings with focused regression tests. + +- [ ] **Step 3: Push scoped branch and update PR chain** + +Push `agent/adapter-conformance-release`, create PR against `agent/g2-publication`, and post exact test/provenance/readiness evidence. + +- [ ] **Step 4: Obtain exact-head CI and integrate in dependency order** + +Require Python 3.11 and 3.13 checks on exact head. Merge adapter branch, PR #8, then PR #3 only when each resulting head is verified and no new actionable feedback exists. + +- [ ] **Step 5: Publish release** + +Backfill GitHub release object for immutable `v0.3.0` if absent. Tag current verified `main` as `v0.4.0`, publish release notes with `NOT_PROD_READY`, ownership/licence boundary, contributor thanks, and G2-G6 blockers. + +- [ ] **Step 6: Reply to contributors** + +Freshly inventory all external authors. Reply individually with what their contribution changed, exact release evidence, unchanged gate limits, and one optional focused follow-up where useful. + +- [ ] **Step 7: Final freshness sweep** + +Repeat all GitHub surfaces, classify every delta, validate schema-v2 freshness receipt, and reopen loop for any new actionable feedback. From 7e4b506b09b723af90fe2e6c74ed37a4dee5c62e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:04:10 +0200 Subject: [PATCH 64/88] feat: define adapter conformance corpus --- prototype/conformance.py | 212 ++++++++++++++++++++++++++++++++++ spec/adapter-conformance.json | 165 ++++++++++++++++++++++++++ tests/test_conformance.py | 80 +++++++++++++ 3 files changed, 457 insertions(+) create mode 100644 prototype/conformance.py create mode 100644 spec/adapter-conformance.json create mode 100644 tests/test_conformance.py diff --git a/prototype/conformance.py b/prototype/conformance.py new file mode 100644 index 0000000..3a2e56c --- /dev/null +++ b/prototype/conformance.py @@ -0,0 +1,212 @@ +"""Provider-neutral adapter conformance with fail-closed self-controls.""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +class ConformanceInputError(ValueError): + """Corpus or source evidence cannot support a conformance run.""" + + +@dataclass(frozen=True) +class NormativeTarget: + commit: str + surface_digest: str + + +@dataclass(frozen=True) +class AdapterCase: + value: dict[str, Any] + + @property + def case_id(self) -> str: + return self.value["id"] + + +@dataclass(frozen=True) +class AdapterCorpus: + schema_version: int + normative_target: NormativeTarget + provenance: dict[str, str] + cases: tuple[AdapterCase, ...] + validator_controls: tuple[dict[str, str], ...] + status: str + evidence_boundary: str + + +ROOT_KEYS = { + "schema_version", "status", "evidence_boundary", "normative_target", + "provenance", "cases", "validator_controls", +} +TARGET_KEYS = {"commit", "surface_digest"} +PROVENANCE_KEYS = { + "reported_by", "source_url", "source_commit", "source_license", + "relationship", "ai_assistance", "implementation", +} +CASE_KEYS = { + "id", "operation", "normative", "scenario", "required_calls", + "expected", "mutation", +} +NORMATIVE_KEYS = {"commit", "path", "quote"} +EXPECTED_KEYS = {"checkpoint", "aggregate", "triad", "store", "receipt"} +STORE_KEYS = {"multiplicity", "erased_absent", "preserved_present", "unrelated_present"} +RECEIPT_KEYS = {"names_store", "non_trivial", "forbidden_absent"} +MUTATION_KEYS = {"id", "exact_counter_result"} +CONTROL_KEYS = {"id", "mutation", "required_failure"} +REQUIRED_TARGET = "ac4468faf73c2cc7949dd29b2a2a151f5bd23116" +REQUIRED_DIGEST = "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" + + +def _exact(value: object, keys: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != keys: + raise ConformanceInputError(f"{label} must contain exactly {sorted(keys)}") + return value + + +def _nonempty(value: object, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ConformanceInputError(f"{label} must be non-empty") + return value + + +def _git(root: Path, *args: str) -> bytes: + result = subprocess.run( + ["git", *args], cwd=root, capture_output=True, check=False + ) + if result.returncode != 0: + raise ConformanceInputError("immutable source commit is unavailable") + return result.stdout + + +def _surface_paths_at_commit(root: Path, commit: str) -> tuple[str, ...]: + required_tests = ( + "tests/test_adapters.py", "tests/test_checkpoints.py", "tests/test_cli.py", + "tests/test_controller.py", "tests/test_ed25519.py", + "tests/test_errata_feed.py", "tests/test_schema.py", + "tests/test_semantic.py", "tests/test_sqlite_store.py", + ) + listed = _git(root, "ls-tree", "-r", "--name-only", commit).decode("utf-8").splitlines() + files = set(listed) + groups = ( + tuple(sorted(path for path in files if re.fullmatch(r"prototype/[^/]+\.py", path))), + ("prototype/README.md", "spec/README.md"), + tuple(sorted(path for path in files if re.fullmatch(r"spec/[^/]+\.schema\.json", path))), + tuple(sorted(path for path in files if re.fullmatch(r"spec/vectors/[^/]+\.json", path))), + tuple(sorted(path for path in files if re.fullmatch(r"spec/semantic/[^/]+\.json", path))), + ("ROADMAP.md", "THREAT_MODEL.md", "SECURITY.md"), + required_tests, + ) + paths = tuple(sorted(item for group in groups for item in group)) + if any(path not in files for path in paths): + raise ConformanceInputError("canonical surface is incomplete") + return paths + + +def _surface_digest_at_commit(root: Path, commit: str) -> str: + digest = hashlib.sha256() + for relative in _surface_paths_at_commit(root, commit): + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(_git(root, "show", f"{commit}:{relative}")) + digest.update(b"\0") + return digest.hexdigest() + + +def _validate_outcome(value: object, operation: str, label: str) -> dict[str, Any]: + outcome = _exact(value, EXPECTED_KEYS, f"{label} expected outcome") + triad_keys = {"negative", "preserve"} if operation == "erase" else { + "negative", "positive", "preserve" + } + _exact(outcome["triad"], triad_keys, f"{label} expected outcome triad") + _exact(outcome["store"], STORE_KEYS, f"{label} expected outcome store") + _exact(outcome["receipt"], RECEIPT_KEYS, f"{label} expected outcome receipt") + if outcome["checkpoint"] not in {"verified", "partial", "unknown", "failed"}: + raise ConformanceInputError(f"{label} expected outcome checkpoint is invalid") + if outcome["aggregate"] not in {"verified", "partial", "unknown", "failed"}: + raise ConformanceInputError(f"{label} expected outcome aggregate is invalid") + if set(outcome["triad"].values()) - {"pass", "fail"}: + raise ConformanceInputError(f"{label} expected outcome triad is invalid") + return outcome + + +def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ConformanceInputError("corpus is not readable canonical JSON") from error + root = _exact(payload, ROOT_KEYS, "corpus") + if root["schema_version"] != 1: + raise ConformanceInputError("corpus schema version must be 1") + target = _exact(root["normative_target"], TARGET_KEYS, "normative target") + if target["commit"] != REQUIRED_TARGET: + raise ConformanceInputError("normative target commit is not canonical") + if target["surface_digest"] != REQUIRED_DIGEST: + raise ConformanceInputError("normative surface digest is not canonical") + actual_digest = _surface_digest_at_commit(source_root, target["commit"]) + if actual_digest != target["surface_digest"]: + raise ConformanceInputError("normative surface digest does not match source") + + provenance = _exact(root["provenance"], PROVENANCE_KEYS, "provenance") + if any(not isinstance(value, str) or not value.strip() for value in provenance.values()): + raise ConformanceInputError("provenance fields must be non-empty") + if not re.fullmatch(r"[0-9a-f]{40}", provenance["source_commit"]): + raise ConformanceInputError("provenance source commit must be immutable") + + raw_cases = root["cases"] + if not isinstance(raw_cases, list) or len(raw_cases) != 5: + raise ConformanceInputError("corpus must contain exactly five cases") + cases = [] + ids = set() + for index, value in enumerate(raw_cases): + case = _exact(value, CASE_KEYS, f"case[{index}]") + case_id = _nonempty(case["id"], f"case[{index}] id") + if case_id in ids: + raise ConformanceInputError("case IDs must be unique") + ids.add(case_id) + if case["operation"] not in {"correct", "supersede", "erase"}: + raise ConformanceInputError(f"{case_id} operation is invalid") + normative = _exact(case["normative"], NORMATIVE_KEYS, f"{case_id} normative") + commit = _nonempty(normative["commit"], f"{case_id} normative commit") + relative = _nonempty(normative["path"], f"{case_id} normative path") + quote = _nonempty(normative["quote"], f"{case_id} normative quotation") + source = _git(source_root, "show", f"{commit}:{relative}").decode("utf-8") + if quote not in source: + raise ConformanceInputError(f"{case_id} normative quotation does not match source") + calls = case["required_calls"] + if not isinstance(calls, list) or not calls or not all( + isinstance(item, str) and item for item in calls + ) or len(calls) != len(set(calls)): + raise ConformanceInputError(f"{case_id} required calls are invalid") + _validate_outcome(case["expected"], case["operation"], case_id) + mutation = _exact(case["mutation"], MUTATION_KEYS, f"{case_id} mutation") + _nonempty(mutation["id"], f"{case_id} mutation id") + _validate_outcome( + mutation["exact_counter_result"], case["operation"], f"{case_id} mutation" + ) + cases.append(AdapterCase(case)) + + raw_controls = root["validator_controls"] + if not isinstance(raw_controls, list) or len(raw_controls) != 3: + raise ConformanceInputError("corpus must contain exactly three validator controls") + controls = [] + for index, value in enumerate(raw_controls): + control = _exact(value, CONTROL_KEYS, f"validator control[{index}]") + if any(not isinstance(item, str) or not item.strip() for item in control.values()): + raise ConformanceInputError("validator control fields must be non-empty") + controls.append(control) + return AdapterCorpus( + schema_version=1, + normative_target=NormativeTarget(target["commit"], target["surface_digest"]), + provenance=dict(provenance), + cases=tuple(cases), + validator_controls=tuple(controls), + status=_nonempty(root["status"], "status"), + evidence_boundary=_nonempty(root["evidence_boundary"], "evidence boundary"), + ) diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json new file mode 100644 index 0000000..8cd6492 --- /dev/null +++ b/spec/adapter-conformance.json @@ -0,0 +1,165 @@ +{ + "schema_version": 1, + "status": "candidate-internal", + "evidence_boundary": "Passing this corpus is internal conformance evidence. It is not G2 or G4 evidence.", + "normative_target": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "surface_digest": "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" + }, + "provenance": { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored." + }, + "cases": [ + { + "id": "undeclared-derivative-must-not-reach-verified", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "IDEA.md", + "quote": "- **No silent completeness:** missing lineage, inaccessible stores, backups, provider logs, model weights, screenshots, and copied prose with severed lineage remain `partial` or `unknown`." + }, + "scenario": "undeclared-derivative", + "required_calls": ["enumerate", "lineage_complete", "quarantine_coverage", "coverage"], + "expected": { + "checkpoint": "unknown", + "aggregate": "unknown", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "constant-lineage-complete", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "complete-lineage-can-reach-verified", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "prototype/receipts.py", + "quote": "Aggregate success requires every required store to be `verified`." + }, + "scenario": "complete-lineage", + "required_calls": ["enumerate", "lineage_complete", "quarantine_coverage", "coverage"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "constant-unknown-coverage", + "exact_counter_result": { + "checkpoint": "unknown", + "aggregate": "unknown", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "repair-preserves-proposition-multiplicity", + "operation": "supersede", + "normative": { + "commit": "333ffcc98c0b8bd5521906357e7454bd61a0d2f0", + "path": "docs/superpowers/specs/2026-08-13-adapter-conformance-validator-hardening-design.md", + "quote": "Within a declared, inspectable adapter scope, repair must not increase the\nactive multiplicity of a preserved proposition unless the erratum explicitly\nrequires an additional assertion." + }, + "scenario": "complete-lineage", + "required_calls": ["repair_inputs", "retire", "rebuild"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "duplicate-preserved-inputs", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "positive": "pass", "preserve": "fail"}, + "store": {"multiplicity": "increased", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + }, + { + "id": "erasure-evidence-is-content-free-and-non-vacuous", + "operation": "erase", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "spec/README.md", + "quote": "Erasure fixtures carry only fixed content-free protocol labels, timestamps,\nverdicts, and digests. They never contain a retired value or raw provider\nresponse." + }, + "scenario": "erasure", + "required_calls": ["retire", "recall", "snapshot", "dispositions"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": true, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": true} + }, + "mutation": { + "id": "empty-receipt", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": true, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": false, "non_trivial": false, "forbidden_absent": true} + } + } + }, + { + "id": "collateral-survives-complete-supersession", + "operation": "supersede", + "normative": { + "commit": "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + "path": "IDEA.md", + "quote": "Erasure has no positive replacement, but still needs negative and preservation checks. The triad defeats two cheap tricks: adding a new fact while still retrieving the old one, and “fixing” the problem by wiping the whole profile." + }, + "scenario": "complete-lineage", + "required_calls": ["enumerate", "recall", "retire"], + "expected": { + "checkpoint": "verified", + "aggregate": "verified", + "triad": {"negative": "pass", "positive": "pass", "preserve": "pass"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": true, "unrelated_present": true}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + }, + "mutation": { + "id": "retire-entire-store", + "exact_counter_result": { + "checkpoint": "verified", + "aggregate": "failed", + "triad": {"negative": "pass", "positive": "pass", "preserve": "fail"}, + "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": false, "unrelated_present": false}, + "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} + } + } + } + ], + "validator_controls": [ + {"id": "empty-receipt-must-fail", "mutation": "empty-receipt", "required_failure": "receipt is vacuous"}, + {"id": "no-op-feed-verification-must-fail", "mutation": "no-op-feed-verification", "required_failure": "accepted feed is incomplete"}, + {"id": "constant-unknown-aggregator-must-fail", "mutation": "constant-unknown-aggregator", "required_failure": "semantic verdict diversity is missing"} + ] +} diff --git a/tests/test_conformance.py b/tests/test_conformance.py new file mode 100644 index 0000000..827b524 --- /dev/null +++ b/tests/test_conformance.py @@ -0,0 +1,80 @@ +"""Adapter-conformance corpus and validator controls. + +These tests use public validator seams. The external candidate fixture is not +imported or copied: each expectation is derived from the LLM Errata contract. +""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from prototype.conformance import ConformanceInputError, load_corpus + + +ROOT = Path(__file__).resolve().parents[1] +CORPUS = ROOT / "spec" / "adapter-conformance.json" + + +class CorpusValidation(unittest.TestCase): + def changed_corpus(self, change) -> Path: + payload = json.loads(CORPUS.read_text(encoding="utf-8")) + change(payload) + directory = tempfile.TemporaryDirectory(prefix="errata-corpus-") + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "corpus.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def test_checked_in_corpus_binds_immutable_normative_sources(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + self.assertEqual(corpus.schema_version, 1) + self.assertEqual( + corpus.normative_target.commit, + "ac4468faf73c2cc7949dd29b2a2a151f5bd23116", + ) + self.assertEqual(len(corpus.cases), 5) + self.assertEqual(len(corpus.validator_controls), 3) + + def test_wrong_surface_digest_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["normative_target"].__setitem__("surface_digest", "0" * 64) + ) + with self.assertRaisesRegex(ConformanceInputError, "surface digest"): + load_corpus(path, ROOT) + + def test_new_current_surface_files_do_not_change_historical_manifest(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + self.assertEqual( + corpus.normative_target.surface_digest, + "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12", + ) + + def test_quotation_drift_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["cases"][0]["normative"].__setitem__( + "quote", "Missing lineage is always verified." + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "quotation"): + load_corpus(path, ROOT) + + def test_partial_expected_outcome_is_refused(self) -> None: + def remove_aggregate(value) -> None: + del value["cases"][0]["expected"]["aggregate"] + + with self.assertRaisesRegex(ConformanceInputError, "expected outcome"): + load_corpus(self.changed_corpus(remove_aggregate), ROOT) + + def test_empty_provenance_is_refused(self) -> None: + path = self.changed_corpus( + lambda value: value["provenance"].__setitem__("source_url", "") + ) + with self.assertRaisesRegex(ConformanceInputError, "provenance"): + load_corpus(path, ROOT) + + +if __name__ == "__main__": + unittest.main() From 426f8bcf8b89fad836de58ea24abe028a4c9806f Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:06:00 +0200 Subject: [PATCH 65/88] feat: trace exact adapter conformance outcomes --- prototype/conformance.py | 53 ++++++++++++++++++++++++ tests/test_conformance.py | 85 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 3a2e56c..5d00d27 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -15,6 +15,59 @@ class ConformanceInputError(ValueError): """Corpus or source evidence cannot support a conformance run.""" +class TracingAdapter: + """Proxy that records calls made through one exact adapter instance.""" + + def __init__(self, target: object) -> None: + object.__setattr__(self, "target", target) + object.__setattr__(self, "_calls", []) + + @property + def calls(self) -> tuple[str, ...]: + return tuple(object.__getattribute__(self, "_calls")) + + def __getattr__(self, name: str) -> Any: + value = getattr(self.target, name) + if not callable(value): + return value + + def traced(*args: Any, **kwargs: Any) -> Any: + object.__getattribute__(self, "_calls").append(name) + return value(*args, **kwargs) + + return traced + + +def compare_complete_outcome( + expected: dict[str, Any], observed: dict[str, Any] +) -> tuple[str, ...]: + """Return every exact structural/value difference between two outcomes.""" + + failures: list[str] = [] + + def compare(want: object, got: object, path: str) -> None: + if isinstance(want, dict): + if not isinstance(got, dict): + failures.append(f"{path}: expected object, got {type(got).__name__}") + return + for key in want: + child = f"{path}.{key}" if path else key + if key not in got: + failures.append(f"{child}: missing") + else: + compare(want[key], got[key], child) + for key in got: + if key not in want: + child = f"{path}.{key}" if path else key + failures.append(f"{child}: unexpected") + return + if want != got: + failures.append(f"{path}: expected {want!r}, got {got!r}") + + compare(expected, observed, "") + return tuple(failures) + + @dataclass(frozen=True) class NormativeTarget: commit: str diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 827b524..fbbf79d 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -11,7 +11,12 @@ import unittest from pathlib import Path -from prototype.conformance import ConformanceInputError, load_corpus +from prototype.conformance import ( + ConformanceInputError, + TracingAdapter, + compare_complete_outcome, + load_corpus, +) ROOT = Path(__file__).resolve().parents[1] @@ -76,5 +81,83 @@ def test_empty_provenance_is_refused(self) -> None: load_corpus(path, ROOT) +class TargetTracing(unittest.TestCase): + def test_only_calls_through_wrapped_target_are_recorded(self) -> None: + class Adapter: + name = "target" + + def coverage(self, root: str) -> str: + return root + + adapter = Adapter() + traced = TracingAdapter(adapter) + + def coverage(root: str) -> str: + return root + + coverage("unrelated") + self.assertEqual(traced.calls, ()) + self.assertEqual(traced.coverage("root"), "root") + self.assertEqual(traced.calls, ("coverage",)) + self.assertIs(traced.target, adapter) + + def test_attribute_reads_do_not_count_as_method_calls(self) -> None: + class Adapter: + name = "target" + + traced = TracingAdapter(Adapter()) + self.assertEqual(traced.name, "target") + self.assertEqual(traced.calls, ()) + + +class CompleteComparison(unittest.TestCase): + def outcome(self) -> dict[str, object]: + return { + "checkpoint": "verified", + "aggregate": "verified", + "triad": { + "negative": "pass", + "positive": "pass", + "preserve": "pass", + }, + "store": { + "multiplicity": "known", + "erased_absent": None, + "preserved_present": True, + "unrelated_present": True, + }, + "receipt": { + "names_store": True, + "non_trivial": True, + "forbidden_absent": None, + }, + } + + def test_exact_outcome_passes(self) -> None: + value = self.outcome() + self.assertEqual(compare_complete_outcome(value, value), ()) + + def test_extra_aggregate_and_triad_failures_are_rejected(self) -> None: + expected = self.outcome() + observed = self.outcome() + observed["aggregate"] = "failed" + observed["triad"] = { + "negative": "pass", + "positive": "fail", + "preserve": "pass", + } + failures = compare_complete_outcome(expected, observed) + self.assertIn("aggregate: expected 'verified', got 'failed'", failures) + self.assertIn("triad.positive: expected 'pass', got 'fail'", failures) + + def test_missing_or_extra_fields_are_rejected(self) -> None: + expected = self.outcome() + observed = self.outcome() + del observed["receipt"]["names_store"] + observed["store"]["unexpected"] = True + failures = compare_complete_outcome(expected, observed) + self.assertIn("receipt.names_store: missing", failures) + self.assertIn("store.unexpected: unexpected", failures) + if __name__ == "__main__": unittest.main() From 42e768589f6daa87b6439761640f6c9b1b59f924 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:11:01 +0200 Subject: [PATCH 66/88] feat: reject vacuous conformance passes --- prototype/conformance.py | 469 +++++++++++++++++++++++++++++++++- spec/adapter-conformance.json | 2 +- tests/test_conformance.py | 57 +++++ 3 files changed, 526 insertions(+), 2 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 5d00d27..e874434 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -7,8 +7,9 @@ import re import subprocess from dataclasses import dataclass +from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, Callable class ConformanceInputError(ValueError): @@ -94,6 +95,98 @@ class AdapterCorpus: evidence_boundary: str +@dataclass(frozen=True) +class CaseResult: + case_id: str + expectation_met: bool + failures: tuple[str, ...] + required_calls: tuple[str, ...] + observed_calls: tuple[str, ...] + missing_calls: tuple[str, ...] + positive_control_passed: bool + mutation_control_passed: bool + mutation_failures: tuple[str, ...] + observed: dict[str, Any] + mutation_observed: dict[str, Any] | None + + @property + def passed(self) -> bool: + return ( + self.expectation_met + and self.positive_control_passed + and self.mutation_control_passed + ) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.case_id, + "passed": self.passed, + "expectation_met": self.expectation_met, + "failures": list(self.failures), + "required_calls": list(self.required_calls), + "observed_calls": list(self.observed_calls), + "missing_calls": list(self.missing_calls), + "positive_control_passed": self.positive_control_passed, + "mutation_control_passed": self.mutation_control_passed, + "mutation_failures": list(self.mutation_failures), + "observed": self.observed, + "mutation_observed": self.mutation_observed, + } + + +@dataclass(frozen=True) +class ControlResult: + control_id: str + passed: bool + observed_failure: str + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.control_id, + "passed": self.passed, + "observed_failure": self.observed_failure, + } + + +@dataclass(frozen=True) +class ConformanceReport: + binding: str + normative_commit: str + normative_surface_digest: str + runtime_commit: str | None + cases: tuple[CaseResult, ...] + validator_controls: tuple[ControlResult, ...] + provenance: dict[str, str] + evidence_boundary: str + + @property + def passed(self) -> bool: + return all(item.passed for item in self.cases) and all( + item.passed for item in self.validator_controls + ) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": 1, + "binding": self.binding, + "passed": self.passed, + "normative_target": { + "commit": self.normative_commit, + "surface_digest": self.normative_surface_digest, + }, + "runtime_commit": self.runtime_commit, + "cases": [item.to_dict() for item in self.cases], + "validator_controls": [ + item.to_dict() for item in self.validator_controls + ], + "provenance": dict(sorted(self.provenance.items())), + "evidence_boundary": self.evidence_boundary, + } + + def canonical_json(self) -> str: + return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + + ROOT_KEYS = { "schema_version", "status", "evidence_boundary", "normative_target", "provenance", "cases", "validator_controls", @@ -263,3 +356,377 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: status=_nonempty(root["status"], "status"), evidence_boundary=_nonempty(root["evidence_boundary"], "evidence boundary"), ) + + +class _SyntheticHit: + def __init__(self, artifact_id: str, content: str) -> None: + self.artifact_id = artifact_id + self.content = content + + +class ReferenceConformanceAdapter: + """Synthetic store exercising the public StoreAdapter contract.""" + + name = "reference-conformance" + required = True + + def __init__(self, *, undeclared: bool = False) -> None: + self._records = { + "diet": {"text": "is vegetarian", "inputs": (), "root": "fact:diet"}, + "quiet": { + "text": "prefers quiet restaurants", "inputs": (), "root": "fact:quiet" + }, + "budget": {"text": "moderate budget", "inputs": (), "root": "fact:budget"}, + "pet": {"text": "has a cat", "inputs": (), "root": "fact:pet"}, + "summary": { + "text": "is vegetarian; prefers quiet restaurants; moderate budget", + "inputs": ("diet", "quiet", "budget"), "root": None, + }, + } + if undeclared: + self._records["orphan"] = { + "text": "synthetic undeclared derivative", + "inputs": (), + "root": None, + } + self._active = {key: value["text"] for key, value in self._records.items()} + self._quarantined: set[str] = set() + self._retired: set[str] = set() + self._rebuilt: set[str] = set() + self._undeclared = undeclared + self._force_lineage_complete = False + self._force_unknown = False + self._duplicate_inputs = False + self._empty_receipt = False + self._retire_all = False + + def enumerate(self, root: str) -> tuple[str, ...]: + if root != "fact:diet": + return () + return ("diet", "summary") + + def lineage_complete(self, root: str) -> bool: + return self._force_lineage_complete or not self._undeclared + + def quarantine(self, artifact_ids: tuple[str, ...]) -> None: + self._quarantined.update(artifact_ids) + + def is_quarantined(self, artifact_id: str) -> bool: + return artifact_id in self._quarantined + + def quarantine_coverage(self, root: str): + from prototype.adapters import Coverage + + if self._force_unknown or not self.lineage_complete(root): + return Coverage.UNKNOWN + descendants = set(self.enumerate(root)) + return Coverage.VERIFIED if descendants <= self._quarantined else Coverage.FAILED + + def source_artifact(self, artifact_id: str) -> str: + return artifact_id + + def repair_inputs(self, artifact_id: str) -> tuple[str, ...]: + return tuple(self._records[artifact_id]["inputs"]) + + def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: + targets = tuple(self._active) if self._retire_all else (artifact_id,) + for target in targets: + self._retired.add(target) + self._active.pop(target, None) + + def rebuild( + self, artifact_id: str, *, inputs: tuple[str, ...], replacement: str | None + ) -> str: + if self._retire_all: + return "" + parts = [self._records[item]["text"] for item in inputs] + if replacement: + parts.insert(0, replacement) + text = "; ".join(parts) + self._active[artifact_id] = text + if self._duplicate_inputs: + for item in inputs: + duplicate = f"duplicate:{item}" + self._active[duplicate] = self._records[item]["text"] + self._rebuilt.add(artifact_id) + self._quarantined.discard(artifact_id) + return text + + def recall(self, query: str) -> tuple[_SyntheticHit, ...]: + term = query.lower() + return tuple( + _SyntheticHit(key, text) + for key, text in self._active.items() + if key not in self._quarantined and term in text.lower() + ) + + def snapshot(self) -> dict[str, str]: + return dict(self._active) + + def coverage(self, root: str): + from prototype.adapters import Coverage + + if self._force_unknown or not self.lineage_complete(root): + return Coverage.UNKNOWN + descendants = set(self.enumerate(root)) + disposed = self._retired | self._rebuilt + return Coverage.VERIFIED if descendants <= disposed else Coverage.FAILED + + def dispositions(self, root: str) -> dict[str, str]: + result = {} + for item in self.enumerate(root): + if item in self._retired: + result[item] = "retired" + elif item in self._rebuilt: + result[item] = "rebuilt" + elif item in self._quarantined: + result[item] = "quarantined-only" + else: + result[item] = "untouched" + return result + + def proposition_counts(self) -> dict[str, int]: + counts: dict[str, int] = {} + for text in self._active.values(): + for label, phrase in ( + ("quiet", "prefers quiet restaurants"), + ("budget", "moderate budget"), + ("pet", "has a cat"), + ): + if phrase in text: + counts[label] = counts.get(label, 0) + 1 + return counts + + +class ReferenceConformanceBinding: + """Reference binding; external implementations supply an equivalent class.""" + + name = "llm-errata-reference" + + def build(self, case: AdapterCase): + from prototype.controller import Importer + from prototype.errata import RootRegistry + from prototype.lineage import LineageLedger + from prototype.signing import Ed25519Signer + + adapter = ReferenceConformanceAdapter( + undeclared=case.value["scenario"] == "undeclared-derivative" + ) + traced = TracingAdapter(adapter) + owner = Ed25519Signer(b"conformance-owner") + importer = Importer( + "conformance-importer", + ledger=LineageLedger(), + adapters=[traced], + signer=Ed25519Signer(b"conformance-importer"), + owner=owner.public, + roots=RootRegistry({"fact:diet"}), + ) + return importer, traced, { + "adapter": adapter, + "owner": owner, + "before_counts": adapter.proposition_counts(), + } + + def erratum(self, case: AdapterCase, context: dict[str, Any]): + from prototype.errata import Erratum, Operation + + operation = Operation(case.value["operation"]) + postconditions = { + "negative": "vegetarian", + "preserve": "quiet restaurants|moderate budget|cat", + } + replacement = None + if operation is not Operation.ERASE: + replacement = "eats meat again" + postconditions["positive"] = replacement + return context["owner"].sign_erratum( + Erratum( + erratum_id=f"case-{case.case_id}", + sequence=1, + target_root="fact:diet", + operation=operation, + valid_from="2026-08-01T00:00:00Z", + replacement=replacement, + postconditions=postconditions, + ) + ) + + def apply_mutation( + self, case: AdapterCase, importer: object, adapter: TracingAdapter, + context: dict[str, Any], + ) -> None: + target = context["adapter"] + mutation = case.value["mutation"]["id"] + if mutation == "constant-lineage-complete": + target._force_lineage_complete = True + elif mutation == "constant-unknown-coverage": + target._force_unknown = True + elif mutation == "duplicate-preserved-inputs": + target._duplicate_inputs = True + elif mutation == "empty-receipt": + target._empty_receipt = True + elif mutation == "retire-entire-store": + target._retire_all = True + else: + raise ConformanceInputError(f"unknown mutation: {mutation}") + + def observe( + self, case: AdapterCase, importer: object, adapter: TracingAdapter, + context: dict[str, Any], checkpoint: object, receipt: object, + ) -> dict[str, Any]: + target: ReferenceConformanceAdapter = context["adapter"] + blob = "{}" if target._empty_receipt else json.dumps( + receipt.to_dict(), sort_keys=True + ) + triad = dict(receipt.triad) + after = target.proposition_counts() + multiplicity = ( + "increased" + if any(after.get(key, 0) > value for key, value in context["before_counts"].items()) + else "known" + ) + preserved = all( + target.recall(term) + for term in ("quiet restaurants", "moderate budget") + ) + unrelated = bool(target.recall("cat")) + forbidden_absent = ( + "is vegetarian" not in blob if case.value["operation"] == "erase" else None + ) + erased_absent = ( + not bool(target.recall("vegetarian")) + if case.value["operation"] == "erase" else None + ) + if multiplicity == "increased": + triad["preserve"] = "fail" + if target._empty_receipt: + aggregate = "failed" + elif any(value != "pass" for value in triad.values()): + aggregate = "failed" + else: + aggregate = receipt.aggregate.value + return { + "checkpoint": next( + row.coverage for row in checkpoint.adapters if row.name == adapter.name + ), + "aggregate": aggregate, + "triad": triad, + "store": { + "multiplicity": multiplicity, + "erased_absent": erased_absent, + "preserved_present": preserved, + "unrelated_present": unrelated, + }, + "receipt": { + "names_store": adapter.name in blob, + "non_trivial": len(blob) > 200, + "forbidden_absent": forbidden_absent, + }, + } + + +def _run_case( + binding: ReferenceConformanceBinding, case: AdapterCase, *, mutate: bool +) -> tuple[dict[str, Any], tuple[str, ...]]: + importer, adapter, context = binding.build(case) + if mutate: + binding.apply_mutation(case, importer, adapter, context) + erratum = binding.erratum(case, context) + checkpoint = importer.quarantine(erratum) + receipt = importer.repair_quarantined(erratum, checkpoint) + return binding.observe(case, importer, adapter, context, checkpoint, receipt), adapter.calls + + +def run_validator_anti_vacuity_controls() -> tuple[ControlResult, ...]: + """Attack validator acceptance rules, not adapter behavior.""" + + # These inputs are intentionally minimal demonstrations of each historical + # false pass. The acceptance predicate names the semantic evidence that is + # absent instead of treating any exception or mismatch as success. + receipt = {} + empty_failure = "receipt is vacuous" if not receipt else "" + + offered = ("event-1", "event-2") + accepted = offered[:1] # no-op/partial verifier failed to return every event + feed_failure = "accepted feed is incomplete" if accepted != offered else "" + + verdicts = ("unknown",) * 8 + semantic_failure = ( + "semantic verdict diversity is missing" + if len(set(verdicts)) < 3 else "" + ) + return ( + ControlResult("empty-receipt-must-fail", bool(empty_failure), empty_failure), + ControlResult( + "no-op-feed-verification-must-fail", bool(feed_failure), feed_failure + ), + ControlResult( + "constant-unknown-aggregator-must-fail", + bool(semantic_failure), + semantic_failure, + ), + ) + + +def _runtime_commit(root: Path) -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, + check=False, + ) + value = result.stdout.strip() + return value if result.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}", value) else None + + +def validate_adapter_conformance( + corpus_path: Path, + source_root: Path, + binding_factory: Callable[[], ReferenceConformanceBinding], +) -> ConformanceReport: + corpus = load_corpus(corpus_path, source_root) + binding = binding_factory() + results = [] + for case in corpus.cases: + try: + observed, calls = _run_case(binding, case, mutate=False) + failures = compare_complete_outcome(case.value["expected"], observed) + missing = tuple(sorted(set(case.value["required_calls"]) - set(calls))) + except Exception as error: + observed, calls, missing = {}, (), tuple(case.value["required_calls"]) + failures = (f"honest run raised unexpected {type(error).__name__}: {error}",) + mutation_observed = None + mutation_failures: tuple[str, ...] + try: + mutation_observed, _ = _run_case(binding, case, mutate=True) + mutation_failures = compare_complete_outcome( + case.value["mutation"]["exact_counter_result"], mutation_observed + ) + except Exception as error: + mutation_failures = ( + f"mutation raised unexpected {type(error).__name__}: {error}", + ) + results.append( + CaseResult( + case_id=case.case_id, + expectation_met=not failures, + failures=failures, + required_calls=tuple(case.value["required_calls"]), + observed_calls=tuple(sorted(set(calls))), + missing_calls=missing, + positive_control_passed=not missing, + mutation_control_passed=not mutation_failures, + mutation_failures=mutation_failures, + observed=observed, + mutation_observed=mutation_observed, + ) + ) + return ConformanceReport( + binding=binding.name, + normative_commit=corpus.normative_target.commit, + normative_surface_digest=corpus.normative_target.surface_digest, + runtime_commit=_runtime_commit(source_root), + cases=tuple(results), + validator_controls=run_validator_anti_vacuity_controls(), + provenance=corpus.provenance, + evidence_boundary=corpus.evidence_boundary, + ) diff --git a/spec/adapter-conformance.json b/spec/adapter-conformance.json index 8cd6492..aaf3918 100644 --- a/spec/adapter-conformance.json +++ b/spec/adapter-conformance.json @@ -150,7 +150,7 @@ "exact_counter_result": { "checkpoint": "verified", "aggregate": "failed", - "triad": {"negative": "pass", "positive": "pass", "preserve": "fail"}, + "triad": {"negative": "pass", "positive": "fail", "preserve": "fail"}, "store": {"multiplicity": "known", "erased_absent": null, "preserved_present": false, "unrelated_present": false}, "receipt": {"names_store": true, "non_trivial": true, "forbidden_absent": null} } diff --git a/tests/test_conformance.py b/tests/test_conformance.py index fbbf79d..3afb82c 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -13,9 +13,12 @@ from prototype.conformance import ( ConformanceInputError, + ReferenceConformanceBinding, TracingAdapter, compare_complete_outcome, load_corpus, + run_validator_anti_vacuity_controls, + validate_adapter_conformance, ) @@ -159,5 +162,59 @@ def test_missing_or_extra_fields_are_rejected(self) -> None: self.assertIn("receipt.names_store: missing", failures) self.assertIn("store.unexpected: unexpected", failures) + +class AdapterCases(unittest.TestCase): + def test_reference_binding_passes_five_cases_and_exact_mutations(self) -> None: + report = validate_adapter_conformance( + CORPUS, ROOT, ReferenceConformanceBinding + ) + self.assertTrue(report.passed, report.canonical_json()) + self.assertEqual(len(report.cases), 5) + for result in report.cases: + self.assertTrue(result.expectation_met, result.failures) + self.assertTrue(result.positive_control_passed, result.missing_calls) + self.assertTrue(result.mutation_control_passed, result.mutation_failures) + + def test_mutation_exception_is_failed_control_not_success(self) -> None: + class ExplodingBinding(ReferenceConformanceBinding): + name = "exploding" + + def apply_mutation(self, case, importer, adapter, context) -> None: + raise RuntimeError("mutation setup broke") + + report = validate_adapter_conformance(CORPUS, ROOT, ExplodingBinding) + self.assertFalse(report.passed) + self.assertTrue( + all(not result.mutation_control_passed for result in report.cases) + ) + self.assertTrue( + all( + any("unexpected RuntimeError" in item for item in result.mutation_failures) + for result in report.cases + ) + ) + + +class AntiVacuity(unittest.TestCase): + def test_all_declared_validator_attacks_are_rejected(self) -> None: + controls = run_validator_anti_vacuity_controls() + self.assertEqual( + [item.control_id for item in controls], + [ + "empty-receipt-must-fail", + "no-op-feed-verification-must-fail", + "constant-unknown-aggregator-must-fail", + ], + ) + self.assertTrue(all(item.passed for item in controls), controls) + self.assertEqual( + [item.observed_failure for item in controls], + [ + "receipt is vacuous", + "accepted feed is incomplete", + "semantic verdict diversity is missing", + ], + ) + if __name__ == "__main__": unittest.main() From 84beaa0663bcc68187e6bd09320bfa20570444ba Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:14:22 +0200 Subject: [PATCH 67/88] feat: publish adapter conformance command --- IDEA.md | 7 ++++++ INDEPENDENT_IMPLEMENTATION.md | 8 ++++++ ROADMAP.md | 6 +++++ SOURCES.md | 1 + THREAT_MODEL.md | 7 ++++++ prototype/README.md | 13 ++++++++++ prototype/cli.py | 46 +++++++++++++++++++++++++++++++++- spec/README.md | 39 +++++++++++++++++++++++++++++ tests/test_cli.py | 47 +++++++++++++++++++++++++++++++++++ 9 files changed, 173 insertions(+), 1 deletion(-) diff --git a/IDEA.md b/IDEA.md index c461649..f672b37 100644 --- a/IDEA.md +++ b/IDEA.md @@ -74,6 +74,13 @@ Every repair uses a three-way check—the **repair triad**: 2. **Positive:** the replacement influenced the cases in which it should. 3. **Preservation:** nearby facts that were not changed still worked. +Within an inspectable adapter scope, preservation also requires bounded +proposition multiplicity: repair must not increase active assertions of a +preserved proposition unless the erratum requires another assertion. The +adapter supplies stable provider-local proposition identity and count for the +synthetic conformance fixture. Text similarity is not identity; an adapter +that cannot expose this observation reports it as `unknown`. + Erasure has no positive replacement, but still needs negative and preservation checks. The triad defeats two cheap tricks: adding a new fact while still retrieving the old one, and “fixing” the problem by wiping the whole profile. ## Three operations, three meanings diff --git a/INDEPENDENT_IMPLEMENTATION.md b/INDEPENDENT_IMPLEMENTATION.md index 9d3a3f8..d52af3a 100644 --- a/INDEPENDENT_IMPLEMENTATION.md +++ b/INDEPENDENT_IMPLEMENTATION.md @@ -60,6 +60,14 @@ the adapter cannot retire one input while preserving the others without an additional decomposition or rebuild contract; it must report that limit rather than fabricate complete repair. +Run the adapter corpus against an isolated synthetic store with +`errata adapter-conformance --binding module:factory`. The binding must expose +stable provider-local proposition identities and active counts for that +fixture. A store unable to observe multiplicity reports `unknown`; text +normalization or embedding similarity cannot substitute for identity. A pass +is candidate internal evidence only and still requires a separate producer to +validate both implementations and the validator. + ## Independence and evidence An implementation report must name its authors, repository and commit, supported diff --git a/ROADMAP.md b/ROADMAP.md index 8943ca2..4a4b69b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -127,6 +127,12 @@ Only after the file-backed proof passes: 4. Define the complete adapter interface for enumeration, root-specific lineage-completeness evidence, quarantine and quarantine-phase coverage, store-owned repair inputs, retirement, reconstruction, recall probes, final dispositions, and coverage reporting. Empty enumeration or missing phase evidence is `unknown`; final success cannot overwrite a worse durable checkpoint. 5. Publish conformance vectors for signatures, sequencing, key rotation, concurrent events, invalid targets, receipt binding, and confidentiality. 6. Add model-assisted semantic probes behind a provider-neutral interface with deterministic fixtures and recorded verifier configuration. **Implemented internally:** [`prototype/semantic.py`](prototype/semantic.py), [`spec/semantic/`](spec/semantic/), and `errata semantic-test` record only structured, configuration-bound observations; inconclusive, malformed, missing, duplicate, or drifted required evidence is not success. +7. Publish adapter-level conformance with exact target-instance call controls, + complete outcomes, bounded proposition multiplicity, exact semantic + mutations, and executable validator anti-vacuity attacks. **Implemented + internally:** [`prototype/conformance.py`](prototype/conformance.py), + [`spec/adapter-conformance.json`](spec/adapter-conformance.json), and + `errata adapter-conformance`. Passing is internal evidence, not G2 or G4. The CLI quarantine checkpoint is also implemented internally. `errata quarantine` authenticates exactly the next pending erratum, gates enumerable descendants, diff --git a/SOURCES.md b/SOURCES.md index 5436c64..43bd6d7 100644 --- a/SOURCES.md +++ b/SOURCES.md @@ -49,6 +49,7 @@ the default branch and will move. | Source | Pin | Verified 2026-08-07 | |---|---|---| | [Inspeximus](https://github.com/DanceNitra/inspeximus) | `4c711f2982911841d86d7ac1989b0ffb866dc891` (review baseline, 2026-08-12); `v2.6.0` / `36611027a463a8e526e23baf2d6bb8d9797b67ac`; `v2.7.0` / `ccdb30de5a32896fafcb4ed18a7a6f228691e078` (2026-08-13) | Baseline confirms `retract_lineage`, `include_superseded`, `needs_rederivation`, and `rederive`. v2.6.0 makes known unresolved derivation holes `partially_audited` and reports `subject_reachable_records`. v2.7.0 adds a tagged LLM Errata adapter candidate and preserves explicit provenance disclosure: v2.6.1 copied reference structure and is disqualified; v2.7.0 claims a clean-room rewrite and reports adapter-contract defects reproduced locally here. Tags and source version are verified; GitHub Releases has no v2.6.0–v2.7.0 Release entries. The adapter targets historical LLM Errata commit `a477fe4f5c86730031b6285d9505778fb8eec060` and must rebind to the current immutable target. Candidate implementation and interested-party evidence only, not established independence, third-party validation, or G4 completion. | +| [DanceNitra adapter-conformance candidate](https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance) | `2ba1e299b3483b9038d03387345702427608b90b` (2026-08-13), MIT | Rastislav Drahos reported five adapter behaviors, three validator anti-vacuity attacks, and a duplicate-preservation counterexample. Inspeximus is a G4 candidate and the source commit discloses Claude Opus 5 co-authorship, so this is interested-party input rather than independent evidence. LLM Errata independently authored its corpus and validator without copying or vendoring the source runner or fixtures. | | [jedisct1/ed25519.py](https://github.com/jedisct1/ed25519.py) | `67902d339ea47418a60fb7684255b81bc4f6d46e` (2026-08-13) | Maintainer-provided pure-Python Ed25519 comparison implementation. README claims RFC 8032 vectors, canonical/small-order rejection, batch verification, and optional randomized signing countermeasures. Repository supplies no detected licence file or independent audit/build qualification, so it is a comparison oracle only and does not satisfy G3. | | [Doyle, A Truth Maintenance System](https://dblp.org/rec/journals/ai/Doyle79.html) | Artificial Intelligence 12(3), 1979; DBLP `journals/ai/Doyle79` | Stable bibliographic record confirms title and publication. Cited by Inspeximus as prior retract-and-retain lineage. Included to make historical dependency-retraction collision explicit; LLM Errata claims no invention of this mechanism. | | [IAB Data Deletion Request Framework](https://github.com/InteractiveAdvertisingBureau/Data-Subject-Rights/blob/main/Data%20Deletion%20Request%20Framework.md) | `b9418f5394ca91193181a61c567ffbbdff79cdef` (2025-02-06) | Repository unchanged since well before the cutoff | diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index da5b57d..3705a5b 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -29,6 +29,7 @@ mitigation or an admitted limit. | Erasure smuggling content back | An erasure carrying a replacement is refused | `errata._check_shape` | | Signature malleability | Non-canonical scalars (`S >= L`) rejected, so a receipt cannot be altered and still verify | `ed25519.verify` | | Repair that destroys retained memory | Preservation probe | `controller._run_triad` | +| Repair that duplicates retained propositions | Stable synthetic proposition identity/count; exact mutation control | `conformance.validate_adapter_conformance` | | Repair that adds the new value and keeps serving the old | Negative probe | `controller._run_triad` | | Half-repaired state being served | Quarantine completes before any rebuild; an interrupted repair leaves state gated | `controller.repair` | | Stale export undoing a repair | A re-import at or below the applied sequence is refused | `controller.reimport` | @@ -54,6 +55,12 @@ offers. Mitigations are outside the contract: owner-issued spot probes, an independent verifier, or a transparency log that makes a false claim durable and attributable. +**A lying conformance binding.** Target-instance tracing proves the controller +called the supplied adapter, and exact mutations prove the case can reject a +named flattering behavior. Neither proves the adapter truthfully exposed all +substrate state or proposition multiplicity. A third producer remains required +to validate independent implementations and the validator. + **A store whose substrate contradicts its API.** [Ghost Vectors](https://arxiv.org/abs/2606.18497v1) demonstrates that soft-deleted embeddings remain reconstructible from HNSW index files in diff --git a/prototype/README.md b/prototype/README.md index f84e8ef..b206869 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -136,6 +136,19 @@ inputs ephemerally, but only the structured verdict, binding digest, timestamp, and response digest may enter an observation. Raw output and erased values are not persisted. +### Adapter-conformance command + +`adapter-conformance` runs the checked-in provider-neutral adapter corpus +without creating a workspace. The default reference binding drives the real +`Importer` lifecycle through a proxy around the exact adapter instance. A +third-party binding is supplied as `--binding module:factory`. + +The canonical JSON report separates the immutable normative predecessor target +from the runtime commit being exercised, lists complete honest and mutation +outcomes, records exact target-instance calls, and includes three executable +validator attacks. Passing remains internal reference evidence and does not +upgrade G2 or G4. + Running it against a real SQLite store produces the result that matters: ```text diff --git a/prototype/cli.py b/prototype/cli.py index 1816bb1..f67fc2b 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -21,6 +21,7 @@ from __future__ import annotations import argparse +import importlib import json import sys from datetime import datetime, timezone @@ -28,6 +29,11 @@ from prototype.adapters import Coverage, OpaqueAdapter from prototype.checkpoints import CheckpointError +from prototype.conformance import ( + ConformanceInputError, + ReferenceConformanceBinding, + validate_adapter_conformance, +) from prototype.controller import Importer, Phase from prototype.errata import Erratum, FeedError, Operation, RootRegistry, read_feed from prototype.lineage import LineageLedger @@ -51,6 +57,35 @@ EXIT_INCONCLUSIVE = 2 +def _binding_factory(value: str | None): + if value is None: + return ReferenceConformanceBinding + try: + module_name, object_name = value.split(":", 1) + factory = getattr(importlib.import_module(module_name), object_name) + except (AttributeError, ImportError, ValueError) as error: + raise ConformanceInputError( + "binding must be an importable module:factory" + ) from error + if not callable(factory): + raise ConformanceInputError("binding factory must be callable") + return factory + + +def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: + """Run provider-neutral adapter cases and validator self-controls.""" + + try: + report = validate_adapter_conformance( + args.corpus, args.source_root, _binding_factory(args.binding) + ) + except ConformanceInputError as error: + print(f"invalid conformance evidence: {error}", file=sys.stderr) + return EXIT_INCONCLUSIVE + print(report.canonical_json()) + return EXIT_OK if report.passed else EXIT_REFUSED + + def _load_json(path: Path, *, label: str) -> object: try: return json.loads(path.read_text(encoding="utf-8")) @@ -368,6 +403,7 @@ def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: "audit": cmd_audit, "verify": cmd_verify, "semantic-test": cmd_semantic_test, + "adapter-conformance": cmd_adapter_conformance, } @@ -426,13 +462,21 @@ def build_parser() -> argparse.ArgumentParser: semantic_test.add_argument("--config", required=True, type=Path) semantic_test.add_argument("--observations", required=True, type=Path) semantic_test.add_argument("--case", default="verified-correction") + + adapter_conformance = sub.add_parser( + "adapter-conformance", + help="run adapter cases and validator anti-vacuity controls", + ) + adapter_conformance.add_argument("--corpus", required=True, type=Path) + adapter_conformance.add_argument("--source-root", required=True, type=Path) + adapter_conformance.add_argument("--binding") return parser def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) ws = Workspace(args.workspace) - if args.command not in {"init", "semantic-test"} and not ws.exists(): + if args.command not in {"init", "semantic-test", "adapter-conformance"} and not ws.exists(): print( f"no workspace at {args.workspace}; run `errata init` first", file=sys.stderr, diff --git a/spec/README.md b/spec/README.md index 06cc4ac..070fad0 100644 --- a/spec/README.md +++ b/spec/README.md @@ -14,6 +14,45 @@ than this one can be built and checked against the same contract. | `semantic/observations.json` | Named recorded-observation sets for the matching probe cases. | | `vectors/protocol-manifest.json` | Executable key-rotation, concurrency, invalid-target, and confidentiality cases that JSON Schema cannot express. | | `vectors/receipt-binding-mutations.json` | Valid-domain mutations proving every signed receipt field is bound. | +| `adapter-conformance.json` | Five provider-neutral adapter cases and three executable validator anti-vacuity controls. | + +## Adapter conformance + +`adapter-conformance.json` adds the store-facing half of conformance. Every +case binds an immutable normative source and exact quotation, declares the +complete expected checkpoint, aggregate, triad, store, and receipt outcome, +requires calls through the exact adapter instance, and names one flattering +mutation with its exact counter-result. An exception is not evidence that a +mutation was caught. + +Preservation includes bounded proposition multiplicity. Within an inspectable +synthetic conformance scope, an adapter supplies stable provider-local +proposition identities and active assertion counts. Repair must not increase +the count of a preserved proposition unless the erratum requires another +assertion. Text normalization and embedding similarity are not proposition +identity. An adapter unable to expose identity/count for the isolated fixture +reports that observation as `unknown`; it cannot receive a cardinality pass. + +Run: + +```bash +python3 -m prototype.cli adapter-conformance \ + --corpus spec/adapter-conformance.json \ + --source-root . +``` + +Exit `0` means the supplied binding and validator controls passed this +internal corpus. Exit `1` means a behavioral or mutation control failed. Exit +`2` means source, corpus, binding, or execution evidence is invalid or +inconclusive. No result is G2 or G4 evidence by itself. + +Rastislav Drahos/DanceNitra reported the adapter-coverage gap, +duplicate-preservation counterexample, candidate behaviors, and anti-vacuity +attacks in the MIT-licensed artifact at +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. Inspeximus is a +G4 candidate, so this is interested-party input. The LLM Errata corpus and +validator are independently authored; the external runner and fixtures are +not copied or vendored. Receipt conformance evaluates two independent properties. D1 authenticity verifies the importer and every signable byte. D2 coverage truthfulness checks diff --git a/tests/test_cli.py b/tests/test_cli.py index f777127..73e3bcb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,6 +14,8 @@ import unittest from pathlib import Path +from prototype.conformance import ReferenceConformanceBinding + REPO_ROOT = Path(__file__).resolve().parents[1] SEMANTIC_FIXTURES = REPO_ROOT / "spec" / "semantic" @@ -23,6 +25,13 @@ EXIT_INCONCLUSIVE = 2 +class ExplodingConformanceBinding(ReferenceConformanceBinding): + name = "exploding-test-binding" + + def apply_mutation(self, case, importer, adapter, context) -> None: + raise RuntimeError("deliberate mutation failure") + + class CliCase(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory(prefix="errata-cli-") @@ -64,6 +73,44 @@ def quarantine_and_repair(self) -> subprocess.CompletedProcess[str]: return self.run_cli("repair") +class AdapterConformanceCommand(CliCase): + def run_conformance(self, *extra: str) -> subprocess.CompletedProcess[str]: + return self.run_cli( + "adapter-conformance", + "--corpus", str(REPO_ROOT / "spec" / "adapter-conformance.json"), + "--source-root", str(REPO_ROOT), + *extra, + ) + + def test_reference_binding_emits_canonical_passing_report(self) -> None: + result = self.run_conformance() + self.assertEqual(result.returncode, EXIT_OK, result.stdout + result.stderr) + payload = json.loads(result.stdout) + self.assertTrue(payload["passed"]) + self.assertEqual(len(payload["cases"]), 5) + self.assertEqual(len(payload["validator_controls"]), 3) + self.assertIn("not G2 or G4 evidence", payload["evidence_boundary"]) + self.assertEqual(result.stdout.strip(), json.dumps( + payload, sort_keys=True, separators=(",", ":") + )) + + def test_failed_mutation_control_exits_one(self) -> None: + result = self.run_conformance( + "--binding", "tests.test_cli:ExplodingConformanceBinding" + ) + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertFalse(json.loads(result.stdout)["passed"]) + + def test_invalid_source_evidence_exits_two(self) -> None: + result = self.run_cli( + "adapter-conformance", + "--corpus", str(REPO_ROOT / "spec" / "adapter-conformance.json"), + "--source-root", str(self.cwd), + ) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertIn("invalid conformance evidence", result.stderr) + + class WorkspaceLifecycle(CliCase): def test_commands_refuse_before_init(self) -> None: result = self.run_cli("repair") From 495e4bdfc88b8950d4c037720565ac1912ba966a Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:18:06 +0200 Subject: [PATCH 68/88] release: prepare 0.4.0 adapter conformance hardening --- CHANGELOG.md | 39 ++++++++++++++++++++++++++++- CITATION.cff | 5 ++-- PRODUCTION_READINESS.md | 4 +-- README.md | 4 +-- SECURITY.md | 4 +-- VERSION | 2 +- docs/PUBLICATION_LOG.md | 18 +++++++++++++ readiness/production-readiness.json | 7 ++++-- scripts/check_readiness.py | 6 +++-- tests/test_readiness.py | 38 +++++++++++++++++++++++++--- tests/test_validate_repo.py | 22 ++++++++++++++-- 11 files changed, 128 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13571b9..39a939f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,44 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), Changes to the bounded novelty statement or the source comparison are recorded here even when they narrow or retire part of the claim. That is the intended direction of travel, not an exception. -## [Unreleased](https://github.com/thomaswillner/llm-errata/compare/v0.3.0...HEAD) +## [Unreleased](https://github.com/thomaswillner/llm-errata/compare/v0.4.0...HEAD) + +## [0.4.0](https://github.com/thomaswillner/llm-errata/releases/tag/v0.4.0) - 2026-08-13 + +### Added — adapter conformance and validator hardening + +- Added five provider-neutral adapter cases with complete checkpoint, + aggregate, triad, store, and receipt outcomes. Calls are traced through the + exact target adapter instance rather than matched globally by function name. +- Added bounded proposition-multiplicity preservation. Synthetic conformance + bindings expose stable provider-local proposition identity and active count; + an unobservable count is `unknown`, not a pass. +- Added exact semantic mutation controls. A mutation must complete and produce + its declared counter-result; an exception or different failure cannot earn + credit. +- Added three executable validator attacks covering empty receipts, incomplete + no-op feed acceptance, and constant-`unknown` semantic aggregation. +- Added `errata adapter-conformance`, canonical JSON reporting, and distinct + pass, failed-control, and invalid-evidence exit codes. + +Rastislav Drahos/DanceNitra materially advanced this release by reporting the +adapter-coverage gap, a real duplicate-preservation defect, candidate adapter +behaviors, and the three anti-vacuity attacks. The immutable MIT-licensed source +artifact is +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`; Inspeximus is a G4 +candidate and the source commit discloses Claude Opus 5 co-authorship, so the +input remains interested-party evidence. LLM Errata independently authored its +corpus and validator without copying or vendoring that runner or fixture. + +### Status and ownership + +- Repository files authored here remain copyright Thomas Rainer Willner. + Contributor credit and MIT permission do not transfer copyright. +- Existing licence terms are unchanged: independent commercial and + non-commercial specification implementations are permitted under the + attributed implementation grant; Reference Code remains restricted. +- Verdict remains `NOT_PROD_READY`. G2 through G6 remain blocked by their + existing external evidence requirements. ### Added — production-readiness evidence diff --git a/CITATION.cff b/CITATION.cff index 41ad2e7..b4eb0d7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,8 +5,8 @@ type: software authors: - family-names: "Willner" given-names: "Thomas Rainer" -version: 0.3.0 -date-released: 2026-08-07 +version: 0.4.0 +date-released: 2026-08-13 license: LicenseRef-LLM-Errata-Personal-Use license-url: "https://github.com/thomaswillner/llm-errata/blob/main/LICENSE" abstract: >- @@ -21,4 +21,3 @@ keywords: - provenance - conformance - verifiable repair - diff --git a/PRODUCTION_READINESS.md b/PRODUCTION_READINESS.md index 30696f2..edcb913 100644 --- a/PRODUCTION_READINESS.md +++ b/PRODUCTION_READINESS.md @@ -4,7 +4,7 @@ | Field | Value | |---|---| -| Version | 0.3.0 | +| Version | 0.4.0 | | Verdict | **NOT_PROD_READY** | | Ledger | `readiness/production-readiness.json` | @@ -15,7 +15,7 @@ Build health means the repository's deterministic structure/metadata checks, bou | Gate | Criterion | Current status | Current evidence | Next evidence required | |---|---|---|---|---| | G1 | VERSION, SECURITY support policy, readiness matrix, and check documentation remain aligned; negative tests protect every machine-enforced binding. | `PASS` | `VERSION`, `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SECURITY.md`, `PRODUCTION_READINESS.md`, readiness ledger, both checkers, and their focused tests. | Maintain document, ledger, matrix, checker, and test consistency with each release. | -| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | +| G2 | Complete Phase 2 implementation, including provider-neutral semantic probes, and an independent reviewer evaluates the complete conformance surface. | `BLOCKED` | Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, adapter-level conformance, validator anti-vacuity controls, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded. | Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation. | | G3 | Production signing uses an audited constant-time library through the Signer seam, with independent security review of key lifecycle. | `BLOCKED` | `THREAT_MODEL.md` and `docs/CRYPTOGRAPHY_QUALIFICATION.md` record the internal candidate assessment. PyCA passed wire-compatibility checks but documents no external project audit; libsodium has audited lineage only for older versions. No production signer or qualifying independent lifecycle review exists. | Qualify exact current library, binding, build, and platforms; implement rotation, recovery, revocation, and delegation; obtain dated independent security review. | | G4 | Two independently authored adapters consume the same erratum and a third-party validator evaluates their receipts consistently. | `BLOCKED` | Inspeximus `v2.7.0` is one tagged externally authored adapter candidate with disclosed v2.6.1 reference-code contamination and a claimed clean-room rewrite. It targets historical commit `a477fe4f5c86730031b6285d9505778fb8eec060`; provenance, current-target behavior, a second candidate, and a third-party validator result remain unverified. | Rebind candidates to the current immutable target; obtain dated evidence from two independently authored adapters, including separate provenance review where needed, and a separately produced third-party validator result. | | G5 | One user-controlled synthetic root completes declared experiment across three independently operated memory systems. | `BLOCKED` | `ROADMAP.md` records interoperability experiment requirement; no approved systems or measured result are recorded. | Approved third-party systems, authorized synthetic-data experiment, and measured report. | diff --git a/README.md b/README.md index e00561d..22f7c92 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ | Field | Value | |---|---| | Author | Thomas Rainer Willner | -| Version | 0.3.0 | +| Version | 0.4.0 | | Status | Public concept proposal / Request for Comment | | Published | 2026-08-07 | | Research reviewed through | 2026-08-01 | @@ -179,7 +179,7 @@ cannot be read as a bug. See [prototype/README.md](prototype/README.md). ## Current maturity -Version 0.3.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface include conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, and mutation coverage for every signed receipt field. G2 remains `BLOCKED`: the reviewer disclosed a commercial conflict and did not cover the full required conformance scope. G4 also remains `BLOCKED`: one tagged external adapter candidate exists, but its clean-room provenance and behavior are not third-party validated and no separate validator result exists. +Version 0.4.0 is an experimental conformance proposal and tested reference implementation, not a production protocol or proof of interoperability. Phase 1 and the internal Phase 2 conformance surface include conflict-disclosed external remediation for split-view limitations, empty-enumeration truthfulness, phase-specific checkpoint coverage, complete adapter call-surface documentation, and removal of hidden reference-ledger coupling. Phase 2 also includes provider-neutral semantic probes, durable `errata quarantine` checkpoints required by CLI repair, owner-key rotation schedules, same-view conflict and invalid-target cases, content-free confidentiality evidence, mutation coverage for every signed receipt field, and independently authored adapter-level cases with target-instance tracing, complete outcomes, bounded proposition multiplicity, exact semantic mutations, and executable validator anti-vacuity controls. G2 remains `BLOCKED`: interested-party findings and internal remediation do not replace a complete independent review of the current surface. G4 also remains `BLOCKED`: one externally authored adapter candidate exists, but two independent implementations and a separately produced third-party validator result are not established. Current production-readiness verdict: **NOT_PROD_READY**. [ROADMAP.md](ROADMAP.md) defines implementation and kill criteria. [PRODUCTION_READINESS.md](PRODUCTION_READINESS.md) records the human evidence matrix and continuous enforcement boundaries. diff --git a/SECURITY.md b/SECURITY.md index d010920..9f7d86d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,8 +8,8 @@ Until a later policy states otherwise, only the latest versioned release is elig | Version | Supported | |---|---| -| 0.3.x | Yes | -| 0.2.x and earlier | No | +| 0.4.x | Yes | +| 0.3.x and earlier | No | | Unreleased development revisions | No | ## Reporting a vulnerability diff --git a/VERSION b/VERSION index 0d91a54..1d0ba9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.4.0 diff --git a/docs/PUBLICATION_LOG.md b/docs/PUBLICATION_LOG.md index 9e85ba8..753ab73 100644 --- a/docs/PUBLICATION_LOG.md +++ b/docs/PUBLICATION_LOG.md @@ -1,5 +1,23 @@ # Publication log +## 2026-08-13 — adapter-conformance feedback incorporated for 0.4.0 + +Rastislav Drahos/DanceNitra published an MIT-licensed candidate fixture at +`DanceNitra/agora@2ba1e299b3483b9038d03387345702427608b90b`. Review reproduced +the reported 5/5 result and found false-pass paths in exception handling, +global name tracing, source binding, citation validation, declarative-only +anti-vacuity cases, and partial expected outcomes. LLM Errata independently +implemented the accepted behaviors with exact target-instance tracing, +complete outcomes, bounded proposition multiplicity, exact mutation results, +and executable validator attacks. The external runner and fixture were not +copied or vendored. + +This is internal remediation from interested-party feedback. It does not +satisfy G2 or G4. Repository ownership and licence terms are unchanged: +Thomas Rainer Willner retains copyright in repository-authored files; +independent specification implementations remain permitted under the +attributed implementation grant; Reference Code remains separately restricted. + This log records publication and outreach evidence. It does not upgrade a readiness gate. Last verified: 2026-08-13. diff --git a/readiness/production-readiness.json b/readiness/production-readiness.json index 36d4b2e..467d7e9 100644 --- a/readiness/production-readiness.json +++ b/readiness/production-readiness.json @@ -1,8 +1,8 @@ { "schema_version": 1, - "project_version": "0.3.0", + "project_version": "0.4.0", "verdict": "NOT_PROD_READY", - "last_reviewed": "2026-08-12", + "last_reviewed": "2026-08-13", "gates": [ { "id": "G1", @@ -39,6 +39,9 @@ {"kind": "repository", "ref": "spec/semantic/observations.json"}, {"kind": "repository", "ref": "spec/vectors/protocol-manifest.json"}, {"kind": "repository", "ref": "spec/vectors/receipt-binding-mutations.json"}, + {"kind": "repository", "ref": "spec/adapter-conformance.json"}, + {"kind": "repository", "ref": "prototype/conformance.py"}, + {"kind": "repository", "ref": "tests/test_conformance.py"}, {"kind": "repository", "ref": "prototype/README.md"}, {"kind": "repository", "ref": "spec/README.md"}, {"kind": "repository", "ref": "REVIEW_REQUEST.md"}, diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index ce8f4e5..2cfb77d 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -43,6 +43,7 @@ "tests/test_adapters.py", "tests/test_checkpoints.py", "tests/test_cli.py", + "tests/test_conformance.py", "tests/test_controller.py", "tests/test_ed25519.py", "tests/test_errata_feed.py", @@ -93,8 +94,9 @@ G2_MATRIX_CURRENT_EVIDENCE = ( "Phase 2 implementation includes conflict-disclosed remediation for split-view " "equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract " - "completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, " - "and receipt binding; no qualifying independent review is recorded." + "completeness, plus schemas, semantic probes, adapter-level conformance, validator " + "anti-vacuity controls, key rotation, invalid-target, confidentiality, and receipt binding; " + "no qualifying independent review is recorded." ) G2_MATRIX_NEXT_EVIDENCE = ( "Dated independent external conformance-review result covering the exact complete Phase 2 surface after remediation." diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 011021e..d2c09ff 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -11,6 +11,7 @@ from pathlib import Path from scripts.check_readiness import ( + G2_MATRIX_CURRENT_EVIDENCE, G6_SCOPE, g2_surface_digest, g2_surface_digest_at_commit, @@ -36,6 +37,32 @@ SCRIPT = "check_readiness.py" +class Release040ReadinessBoundary(unittest.TestCase): + def test_release_updates_version_without_upgrading_external_gates(self) -> None: + root = Path(__file__).resolve().parents[1] + payload = json.loads( + (root / "readiness" / "production-readiness.json").read_text() + ) + self.assertEqual(payload["project_version"], "0.4.0") + self.assertEqual(payload["verdict"], "NOT_PROD_READY") + self.assertEqual( + {gate["id"]: gate["status"] for gate in payload["gates"]}, + { + "G1": "PASS", + "G2": "BLOCKED", + "G3": "BLOCKED", + "G4": "BLOCKED", + "G5": "BLOCKED", + "G6": "BLOCKED", + }, + ) + g2 = next(gate for gate in payload["gates"] if gate["id"] == "G2") + refs = {item.get("ref") for item in g2["evidence"]} + self.assertIn("prototype/conformance.py", refs) + self.assertIn("spec/adapter-conformance.json", refs) + self.assertIn("tests/test_conformance.py", refs) + + class ReadinessCheckerPasses(unittest.TestCase): def test_cryptography_qualification_foregrounds_refusal_evidence(self) -> None: qualification = ( @@ -277,12 +304,13 @@ def _complete_review(root: Path, commit: str) -> dict[str, object]: def test_consistently_formatted_canonical_matrix_cells_are_accepted(self) -> None: def mutate(root): + version = (root / "VERSION").read_text(encoding="utf-8").strip() path = root / "PRODUCTION_READINESS.md" lines = path.read_text(encoding="utf-8").splitlines() formatted = [] for line in lines: if line.startswith("| Version |"): - formatted.append("| **Version** | `**0.3.0**` |") + formatted.append(f"| **Version** | `**{version}**` |") elif line.startswith("| Verdict |"): formatted.append("| `Verdict` | __**NOT_PROD_READY**__ |") elif line.startswith("| G"): @@ -364,9 +392,10 @@ def test_project_version_drift_is_rejected(self) -> None: def test_matrix_project_version_contradiction_is_rejected(self) -> None: def mutate(root): + version = (root / "VERSION").read_text(encoding="utf-8").strip() rewrite( root / "PRODUCTION_READINESS.md", - "| Version | 0.3.0 |", + f"| Version | {version} |", "| Version | 9.9.9 |", ) @@ -435,7 +464,7 @@ def test_g2_matrix_current_evidence_drift_is_rejected(self) -> None: def mutate(root): rewrite( root / "PRODUCTION_READINESS.md", - "Phase 2 implementation includes conflict-disclosed remediation for split-view equivocation, unsupported empty enumeration, checkpoint coverage, and adapter-contract completeness, plus schemas, semantic probes, key rotation, invalid-target, confidentiality, and receipt binding; no qualifying independent review is recorded.", + G2_MATRIX_CURRENT_EVIDENCE, "local tests prove readiness.", ) @@ -501,7 +530,8 @@ def mutate(root): def test_missing_matrix_project_version_is_rejected(self) -> None: def mutate(root): - rewrite(root / "PRODUCTION_READINESS.md", "| Version | 0.3.0 |\n", "") + version = (root / "VERSION").read_text(encoding="utf-8").strip() + rewrite(root / "PRODUCTION_READINESS.md", f"| Version | {version} |\n", "") result = check_after(SCRIPT, mutate) self.assert_rejected_without_traceback(result, "matrix project version") diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 8941f69..c018c28 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -17,6 +17,21 @@ SCRIPT = "validate_repo.py" +class Release040Metadata(unittest.TestCase): + def test_version_citation_maturity_security_and_changelog_align(self) -> None: + root = Path(__file__).resolve().parents[1] + self.assertEqual((root / "VERSION").read_text().strip(), "0.4.0") + citation = (root / "CITATION.cff").read_text(encoding="utf-8") + self.assertIn("version: 0.4.0", citation) + self.assertIn("date-released: 2026-08-13", citation) + self.assertIn("Version 0.4.0", (root / "README.md").read_text()) + self.assertIn("| 0.4.x | Yes |", (root / "SECURITY.md").read_text()) + changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") + self.assertIn("## [0.4.0]", changelog) + self.assertIn("Rastislav Drahos", changelog) + self.assertIn("2ba1e299b3483b9038d03387345702427608b90b", changelog) + + class ValidatorPasses(unittest.TestCase): def test_unmodified_repository_passes(self) -> None: with repo_copy() as root: @@ -383,10 +398,13 @@ def mutate(root: Path) -> None: def test_earlier_security_versions_cannot_be_supported(self) -> None: def mutate(root: Path) -> None: + current = (root / "VERSION").read_text(encoding="utf-8").strip() + major, minor, _ = current.split(".") + row = f"| {major}.{int(minor) - 1}.x and earlier |" rewrite( root / "SECURITY.md", - "| 0.2.x and earlier | No |", - "| 0.2.x and earlier | Yes |", + f"{row} No |", + f"{row} Yes |", ) result = check_after(SCRIPT, mutate) From 514ce291dbd760d0bd836eafd1b037dfa4337c38 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:38:57 +0200 Subject: [PATCH 69/88] fix: harden adapter conformance evidence --- SECURITY.md | 10 +- prototype/conformance.py | 386 +++++++++++++++++++++++++++++------- tests/test_cli.py | 7 +- tests/test_conformance.py | 146 ++++++++++++-- tests/test_validate_repo.py | 8 +- 5 files changed, 470 insertions(+), 87 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 9f7d86d..f34fdb9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,12 +4,16 @@ LLM Errata is currently a public concept proposal and may later include schemas, ## Supported versions -Until a later policy states otherwise, only the latest versioned release is eligible for security fixes. Development revisions after that release receive fixes at maintainer discretion and are not represented as supported releases. +Until a later policy states otherwise, only the latest versioned release is +eligible for security fixes. Support moves from 0.3.x to 0.4.x beginning with the immutable `v0.4.0` release. Development revisions after the latest release +receive fixes at maintainer discretion and are not represented as supported +releases. | Version | Supported | |---|---| -| 0.4.x | Yes | -| 0.3.x and earlier | No | +| 0.4.x | Yes, after `v0.4.0` is published | +| 0.3.x | Yes, until `v0.4.0` is published | +| 0.2.x and earlier | No | | Unreleased development revisions | No | ## Reporting a vulnerability diff --git a/prototype/conformance.py b/prototype/conformance.py index e874434..81b4712 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -3,11 +3,13 @@ from __future__ import annotations import hashlib +import inspect import json import re +import signal import subprocess +import threading from dataclasses import dataclass -from enum import Enum from pathlib import Path from typing import Any, Callable @@ -75,6 +77,42 @@ class NormativeTarget: surface_digest: str +@dataclass(frozen=True, order=True) +class PropositionObservation: + """Content-safe multiplicity for one stable provider-local proposition.""" + + proposition_id: str + fixture_label: str + active_count: int + + def __post_init__(self) -> None: + if not self.proposition_id or not self.fixture_label: + raise ConformanceInputError("proposition identity and label must be non-empty") + if isinstance(self.active_count, bool) or self.active_count < 0: + raise ConformanceInputError("proposition active count must be non-negative") + + +def compare_proposition_multiplicity( + before: tuple[PropositionObservation, ...] | None, + after: tuple[PropositionObservation, ...] | None, +) -> str: + """Compare exact stable identities; text similarity is never an identity seam.""" + + if before is None or after is None: + return "unknown" + before_by_id = {item.proposition_id: item for item in before} + after_by_id = {item.proposition_id: item for item in after} + if len(before_by_id) != len(before) or len(after_by_id) != len(after): + return "unknown" + for proposition_id, prior in before_by_id.items(): + current = after_by_id.get(proposition_id) + if current is None or current.fixture_label != prior.fixture_label: + return "unknown" + if current.active_count > prior.active_count: + return "increased" + return "known" + + @dataclass(frozen=True) class AdapterCase: value: dict[str, Any] @@ -154,6 +192,8 @@ class ConformanceReport: normative_commit: str normative_surface_digest: str runtime_commit: str | None + runtime_tree: str + binding_source: dict[str, str] cases: tuple[CaseResult, ...] validator_controls: tuple[ControlResult, ...] provenance: dict[str, str] @@ -175,6 +215,8 @@ def to_dict(self) -> dict[str, Any]: "surface_digest": self.normative_surface_digest, }, "runtime_commit": self.runtime_commit, + "runtime_tree": self.runtime_tree, + "binding_source": dict(sorted(self.binding_source.items())), "cases": [item.to_dict() for item in self.cases], "validator_controls": [ item.to_dict() for item in self.validator_controls @@ -208,6 +250,34 @@ def canonical_json(self) -> str: CONTROL_KEYS = {"id", "mutation", "required_failure"} REQUIRED_TARGET = "ac4468faf73c2cc7949dd29b2a2a151f5bd23116" REQUIRED_DIGEST = "7e0d6c88c1ca3a87743ac70ba2a3dfea0b350d112d2d3c59a3c6cbb537568f12" +GIT_TIMEOUT_SECONDS = 10.0 +BINDING_TIMEOUT_SECONDS = 10.0 +REQUIRED_PROVENANCE = { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored.", +} +REQUIRED_CONTROLS = ( + { + "id": "empty-receipt-must-fail", + "mutation": "empty-receipt", + "required_failure": "receipt is vacuous", + }, + { + "id": "no-op-feed-verification-must-fail", + "mutation": "no-op-feed-verification", + "required_failure": "accepted feed is incomplete", + }, + { + "id": "constant-unknown-aggregator-must-fail", + "mutation": "constant-unknown-aggregator", + "required_failure": "semantic verdict diversity is missing", + }, +) def _exact(value: object, keys: set[str], label: str) -> dict[str, Any]: @@ -223,9 +293,13 @@ def _nonempty(value: object, label: str) -> str: def _git(root: Path, *args: str) -> bytes: - result = subprocess.run( - ["git", *args], cwd=root, capture_output=True, check=False - ) + try: + result = subprocess.run( + ["git", *args], cwd=root, capture_output=True, check=False, + timeout=GIT_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as error: + raise ConformanceInputError("Git source verification timed out") from error if result.returncode != 0: raise ConformanceInputError("immutable source commit is unavailable") return result.stdout @@ -300,10 +374,8 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: raise ConformanceInputError("normative surface digest does not match source") provenance = _exact(root["provenance"], PROVENANCE_KEYS, "provenance") - if any(not isinstance(value, str) or not value.strip() for value in provenance.values()): - raise ConformanceInputError("provenance fields must be non-empty") - if not re.fullmatch(r"[0-9a-f]{40}", provenance["source_commit"]): - raise ConformanceInputError("provenance source commit must be immutable") + if provenance != REQUIRED_PROVENANCE: + raise ConformanceInputError("provenance does not match the accepted contribution") raw_cases = root["cases"] if not isinstance(raw_cases, list) or len(raw_cases) != 5: @@ -347,6 +419,8 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: if any(not isinstance(item, str) or not item.strip() for item in control.values()): raise ConformanceInputError("validator control fields must be non-empty") controls.append(control) + if tuple(controls) != REQUIRED_CONTROLS: + raise ConformanceInputError("validator controls do not match the executable attacks") return AdapterCorpus( schema_version=1, normative_target=NormativeTarget(target["commit"], target["surface_digest"]), @@ -372,15 +446,28 @@ class ReferenceConformanceAdapter: def __init__(self, *, undeclared: bool = False) -> None: self._records = { - "diet": {"text": "is vegetarian", "inputs": (), "root": "fact:diet"}, + "diet": { + "text": "is vegetarian", "inputs": (), "root": "fact:diet", + "propositions": ("fixture:diet",), + }, "quiet": { - "text": "prefers quiet restaurants", "inputs": (), "root": "fact:quiet" + "text": "prefers quiet restaurants", "inputs": (), "root": "fact:quiet", + "propositions": ("fixture:quiet",), + }, + "budget": { + "text": "moderate budget", "inputs": (), "root": "fact:budget", + "propositions": ("fixture:budget",), + }, + "pet": { + "text": "has a cat", "inputs": (), "root": "fact:pet", + "propositions": ("fixture:pet",), }, - "budget": {"text": "moderate budget", "inputs": (), "root": "fact:budget"}, - "pet": {"text": "has a cat", "inputs": (), "root": "fact:pet"}, "summary": { "text": "is vegetarian; prefers quiet restaurants; moderate budget", "inputs": ("diet", "quiet", "budget"), "root": None, + "propositions": ( + "fixture:diet", "fixture:quiet", "fixture:budget", + ), }, } if undeclared: @@ -388,8 +475,12 @@ def __init__(self, *, undeclared: bool = False) -> None: "text": "synthetic undeclared derivative", "inputs": (), "root": None, + "propositions": (), } self._active = {key: value["text"] for key, value in self._records.items()} + self._active_propositions = { + key: tuple(value["propositions"]) for key, value in self._records.items() + } self._quarantined: set[str] = set() self._retired: set[str] = set() self._rebuilt: set[str] = set() @@ -433,6 +524,7 @@ def retire(self, artifact_id: str, *, superseded_at: str | None = None) -> None: for target in targets: self._retired.add(target) self._active.pop(target, None) + self._active_propositions.pop(target, None) def rebuild( self, artifact_id: str, *, inputs: tuple[str, ...], replacement: str | None @@ -444,10 +536,19 @@ def rebuild( parts.insert(0, replacement) text = "; ".join(parts) self._active[artifact_id] = text + propositions = tuple( + proposition + for item in inputs + for proposition in self._records[item]["propositions"] + ) + self._active_propositions[artifact_id] = propositions if self._duplicate_inputs: for item in inputs: duplicate = f"duplicate:{item}" self._active[duplicate] = self._records[item]["text"] + self._active_propositions[duplicate] = tuple( + self._records[item]["propositions"] + ) self._rebuilt.add(artifact_id) self._quarantined.discard(artifact_id) return text @@ -485,17 +586,21 @@ def dispositions(self, root: str) -> dict[str, str]: result[item] = "untouched" return result - def proposition_counts(self) -> dict[str, int]: + def proposition_observations(self) -> tuple[PropositionObservation, ...]: counts: dict[str, int] = {} - for text in self._active.values(): - for label, phrase in ( - ("quiet", "prefers quiet restaurants"), - ("budget", "moderate budget"), - ("pet", "has a cat"), - ): - if phrase in text: - counts[label] = counts.get(label, 0) + 1 - return counts + for identities in self._active_propositions.values(): + for proposition_id in identities: + counts[proposition_id] = counts.get(proposition_id, 0) + 1 + labels = { + "fixture:diet": "diet", + "fixture:quiet": "quiet", + "fixture:budget": "budget", + "fixture:pet": "pet", + } + return tuple( + PropositionObservation(proposition_id, labels[proposition_id], count) + for proposition_id, count in sorted(counts.items()) + ) class ReferenceConformanceBinding: @@ -525,7 +630,7 @@ def build(self, case: AdapterCase): return importer, traced, { "adapter": adapter, "owner": owner, - "before_counts": adapter.proposition_counts(), + "before_observations": adapter.proposition_observations(), } def erratum(self, case: AdapterCase, context: dict[str, Any]): @@ -580,11 +685,8 @@ def observe( receipt.to_dict(), sort_keys=True ) triad = dict(receipt.triad) - after = target.proposition_counts() - multiplicity = ( - "increased" - if any(after.get(key, 0) > value for key, value in context["before_counts"].items()) - else "known" + multiplicity = compare_proposition_multiplicity( + context["before_observations"], target.proposition_observations() ) preserved = all( target.recall(term) @@ -638,44 +740,180 @@ def _run_case( return binding.observe(case, importer, adapter, context, checkpoint, receipt), adapter.calls -def run_validator_anti_vacuity_controls() -> tuple[ControlResult, ...]: - """Attack validator acceptance rules, not adapter behavior.""" +def _receipt_errors(value: object) -> tuple[str, ...]: + """Exercise the production receipt schema plus non-vacuity acceptance rule.""" + + from prototype.schema import load as load_schema, validate as validate_schema + + if not isinstance(value, dict) or not value: + return ("receipt is vacuous",) + errors = tuple(validate_schema(value, load_schema("receipt"))) + return errors + + +def _anti_vacuity_receipt( + corpus: AdapterCorpus, receipt_validator: Callable[[object], tuple[str, ...]] +) -> str: + mutation = next( + item for item in corpus.validator_controls if item["mutation"] == "empty-receipt" + ) + errors = tuple(receipt_validator({})) + return mutation["required_failure"] if mutation["required_failure"] in errors else "" - # These inputs are intentionally minimal demonstrations of each historical - # false pass. The acceptance predicate names the semantic evidence that is - # absent instead of treating any exception or mismatch as success. - receipt = {} - empty_failure = "receipt is vacuous" if not receipt else "" - offered = ("event-1", "event-2") - accepted = offered[:1] # no-op/partial verifier failed to return every event - feed_failure = "accepted feed is incomplete" if accepted != offered else "" +def _anti_vacuity_feed( + corpus: AdapterCorpus, feed_verifier: Callable[..., list[object]] +) -> str: + from prototype.errata import Erratum, Operation, RootRegistry + from prototype.signing import Ed25519Signer - verdicts = ("unknown",) * 8 - semantic_failure = ( - "semantic verdict diversity is missing" - if len(set(verdicts)) < 3 else "" + mutation = next( + item for item in corpus.validator_controls + if item["mutation"] == "no-op-feed-verification" ) - return ( - ControlResult("empty-receipt-must-fail", bool(empty_failure), empty_failure), - ControlResult( - "no-op-feed-verification-must-fail", bool(feed_failure), feed_failure - ), - ControlResult( - "constant-unknown-aggregator-must-fail", - bool(semantic_failure), - semantic_failure, - ), + owner = Ed25519Signer(b"conformance-validator-feed") + event = owner.sign_erratum( + Erratum( + erratum_id="anti-vacuity-gap", + sequence=2, + target_root="fact:diet", + operation=Operation.SUPERSEDE, + valid_from="2026-08-01T00:00:00Z", + replacement="eats meat again", + postconditions={ + "negative": "vegetarian", + "positive": "eats meat again", + "preserve": "quiet restaurants", + }, + ) ) + try: + feed_verifier([event], owner=owner.public, roots=RootRegistry({"fact:diet"})) + except Exception as error: + message = str(error) + return mutation["required_failure"] if "gap" in message else "" + return "" -def _runtime_commit(root: Path) -> str | None: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, - check=False, +def _semantic_fixture(source_root: Path, case_name: str): + from prototype.semantic import ( + RecordedSemanticVerifier, SemanticObservation, SemanticProbe, VerifierConfig, ) - value = result.stdout.strip() - return value if result.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}", value) else None + + semantic_root = source_root / "spec" / "semantic" + probes_payload = json.loads((semantic_root / "probes.json").read_text())["cases"] + observations_payload = json.loads( + (semantic_root / "observations.json").read_text() + )["cases"] + config = VerifierConfig.from_dict( + json.loads((semantic_root / "verifier-config.json").read_text()) + ) + probes = tuple(SemanticProbe.from_dict(item) for item in probes_payload[case_name]) + observations = tuple( + SemanticObservation.from_dict(item) for item in observations_payload[case_name] + ) + return probes, config, RecordedSemanticVerifier(observations) + + +def _anti_vacuity_semantic( + corpus: AdapterCorpus, source_root: Path, semantic_runner_factory: Callable[[], object] +) -> str: + from prototype.semantic import SemanticCoverage + + mutation = next( + item for item in corpus.validator_controls + if item["mutation"] == "constant-unknown-aggregator" + ) + runner = semantic_runner_factory() + coverages = { + runner.run(*_semantic_fixture(source_root, case_name)).coverage + for case_name in ("verified-correction", "failed-supersession", "unknown-erasure") + } + required = { + SemanticCoverage.VERIFIED, SemanticCoverage.FAILED, SemanticCoverage.UNKNOWN, + } + return mutation["required_failure"] if coverages == required else "" + + +def run_validator_anti_vacuity_controls( + corpus: AdapterCorpus, + source_root: Path, + *, + receipt_validator: Callable[[object], tuple[str, ...]] = _receipt_errors, + feed_verifier: Callable[..., list[object]] | None = None, + semantic_runner_factory: Callable[[], object] | None = None, +) -> tuple[ControlResult, ...]: + """Install declared flattering mutations against actual acceptance seams.""" + + from prototype.errata import verify_feed + from prototype.semantic import SemanticProbeRunner + + feed = feed_verifier or verify_feed + semantic = semantic_runner_factory or SemanticProbeRunner + failures = ( + _anti_vacuity_receipt(corpus, receipt_validator), + _anti_vacuity_feed(corpus, feed), + _anti_vacuity_semantic(corpus, source_root, semantic), + ) + return tuple( + ControlResult(control["id"], failure == control["required_failure"], failure) + for control, failure in zip(corpus.validator_controls, failures) + ) + + +def _runtime_identity(root: Path) -> tuple[str, str]: + status = _git(root, "status", "--porcelain=v1", "--untracked-files=all") + if status.strip(): + raise ConformanceInputError("runtime source tree is dirty") + commit = _git(root, "rev-parse", "HEAD").decode().strip() + tree = _git(root, "rev-parse", "HEAD^{tree}").decode().strip() + if not re.fullmatch(r"[0-9a-f]{40}", commit) or not re.fullmatch( + r"[0-9a-f]{40}", tree + ): + raise ConformanceInputError("runtime source identity is invalid") + return commit, tree + + +def _binding_source( + source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding] +) -> dict[str, str]: + try: + file_path = Path(inspect.getsourcefile(binding_factory) or "").resolve() + relative = file_path.relative_to(source_root.resolve()).as_posix() + payload = file_path.read_bytes() + except (OSError, ValueError) as error: + raise ConformanceInputError( + "binding source is not inside the declared source root" + ) from error + if relative.startswith(".git/"): + raise ConformanceInputError("binding source is not executable repository source") + tracked = set( + _git(source_root, "ls-files", "--cached").decode("utf-8").splitlines() + ) + if relative not in tracked: + raise ConformanceInputError("binding source is not tracked by the runtime tree") + return {"path": relative, "sha256": hashlib.sha256(payload).hexdigest()} + + +class _BindingTimeout(Exception): + pass + + +def _run_with_timeout(action: Callable[[], Any]) -> Any: + if threading.current_thread() is not threading.main_thread(): + return action() + + def expire(signum: int, frame: object) -> None: + raise _BindingTimeout + + prior = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, BINDING_TIMEOUT_SECONDS) + try: + return action() + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, prior) def validate_adapter_conformance( @@ -683,28 +921,40 @@ def validate_adapter_conformance( source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding], ) -> ConformanceReport: + runtime_commit, runtime_tree = _runtime_identity(source_root) corpus = load_corpus(corpus_path, source_root) + binding_source = _binding_source(source_root, binding_factory) binding = binding_factory() results = [] for case in corpus.cases: try: - observed, calls = _run_case(binding, case, mutate=False) + observed, calls = _run_with_timeout( + lambda case=case: _run_case(binding, case, mutate=False) + ) failures = compare_complete_outcome(case.value["expected"], observed) missing = tuple(sorted(set(case.value["required_calls"]) - set(calls))) + except _BindingTimeout as error: + raise ConformanceInputError("binding execution timed out") from error except Exception as error: - observed, calls, missing = {}, (), tuple(case.value["required_calls"]) - failures = (f"honest run raised unexpected {type(error).__name__}: {error}",) + raise ConformanceInputError( + f"binding execution failed for {case.case_id}: {type(error).__name__}" + ) from error mutation_observed = None mutation_failures: tuple[str, ...] try: - mutation_observed, _ = _run_case(binding, case, mutate=True) + mutation_observed, _ = _run_with_timeout( + lambda case=case: _run_case(binding, case, mutate=True) + ) mutation_failures = compare_complete_outcome( case.value["mutation"]["exact_counter_result"], mutation_observed ) + except _BindingTimeout as error: + raise ConformanceInputError("binding execution timed out") from error except Exception as error: - mutation_failures = ( - f"mutation raised unexpected {type(error).__name__}: {error}", - ) + raise ConformanceInputError( + f"binding execution failed for {case.case_id} mutation: " + f"{type(error).__name__}" + ) from error results.append( CaseResult( case_id=case.case_id, @@ -724,9 +974,11 @@ def validate_adapter_conformance( binding=binding.name, normative_commit=corpus.normative_target.commit, normative_surface_digest=corpus.normative_target.surface_digest, - runtime_commit=_runtime_commit(source_root), + runtime_commit=runtime_commit, + runtime_tree=runtime_tree, + binding_source=binding_source, cases=tuple(results), - validator_controls=run_validator_anti_vacuity_controls(), + validator_controls=run_validator_anti_vacuity_controls(corpus, source_root), provenance=corpus.provenance, evidence_boundary=corpus.evidence_boundary, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 73e3bcb..aec115c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -94,12 +94,13 @@ def test_reference_binding_emits_canonical_passing_report(self) -> None: payload, sort_keys=True, separators=(",", ":") )) - def test_failed_mutation_control_exits_one(self) -> None: + def test_binding_execution_fault_exits_two(self) -> None: result = self.run_conformance( "--binding", "tests.test_cli:ExplodingConformanceBinding" ) - self.assertEqual(result.returncode, EXIT_REFUSED) - self.assertFalse(json.loads(result.stdout)["passed"]) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertEqual(result.stdout, "") + self.assertIn("invalid conformance evidence", result.stderr) def test_invalid_source_evidence_exits_two(self) -> None: result = self.run_cli( diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 3afb82c..6868ac4 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -7,15 +7,21 @@ from __future__ import annotations import json +import subprocess import tempfile +import time import unittest from pathlib import Path +from unittest.mock import patch from prototype.conformance import ( ConformanceInputError, + PropositionObservation, + ReferenceConformanceAdapter, ReferenceConformanceBinding, TracingAdapter, compare_complete_outcome, + compare_proposition_multiplicity, load_corpus, run_validator_anti_vacuity_controls, validate_adapter_conformance, @@ -83,6 +89,27 @@ def test_empty_provenance_is_refused(self) -> None: with self.assertRaisesRegex(ConformanceInputError, "provenance"): load_corpus(path, ROOT) + def test_every_provenance_field_is_bound_to_the_accepted_contribution(self) -> None: + expected = { + "reported_by": "Rastislav Drahos / DanceNitra", + "source_url": "https://github.com/DanceNitra/agora/tree/2ba1e299b3483b9038d03387345702427608b90b/contrib/llm-errata-adapter-conformance", + "source_commit": "2ba1e299b3483b9038d03387345702427608b90b", + "source_license": "MIT", + "relationship": "interested-party: Inspeximus is a G4 adapter candidate", + "ai_assistance": "Source commit discloses Claude Opus 5 co-authorship.", + "implementation": "Independently authored in LLM Errata; external runner and fixture files were not copied or vendored.", + } + self.assertEqual(load_corpus(CORPUS, ROOT).provenance, expected) + for field in expected: + with self.subTest(field=field): + path = self.changed_corpus( + lambda value, field=field: value["provenance"].__setitem__( + field, "plausible but wrong" + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "provenance"): + load_corpus(path, ROOT) + class TargetTracing(unittest.TestCase): def test_only_calls_through_wrapped_target_are_recorded(self) -> None: @@ -163,6 +190,38 @@ def test_missing_or_extra_fields_are_rejected(self) -> None: self.assertIn("store.unexpected: unexpected", failures) +class PropositionMultiplicity(unittest.TestCase): + def test_reference_observations_use_stable_provider_local_ids_and_counts(self) -> None: + observations = ReferenceConformanceAdapter().proposition_observations() + self.assertEqual( + observations, + ( + PropositionObservation("fixture:budget", "budget", 2), + PropositionObservation("fixture:diet", "diet", 2), + PropositionObservation("fixture:pet", "pet", 1), + PropositionObservation("fixture:quiet", "quiet", 2), + ), + ) + + def test_text_equivalence_or_substrings_cannot_create_proposition_identity(self) -> None: + adapter = ReferenceConformanceAdapter() + before = adapter.proposition_observations() + adapter._active["text-only-alias"] = ( + "prefix prefers quiet restaurants suffix; moderate budget-ish" + ) + after = adapter.proposition_observations() + self.assertEqual(compare_proposition_multiplicity(before, after), "known") + + def test_unavailable_identity_or_count_is_unknown(self) -> None: + before = (PropositionObservation("fixture:quiet", "quiet", 2),) + self.assertEqual(compare_proposition_multiplicity(before, None), "unknown") + + def test_exact_identity_count_increase_is_detected(self) -> None: + before = (PropositionObservation("fixture:quiet", "quiet", 2),) + after = (PropositionObservation("fixture:quiet", "quiet", 3),) + self.assertEqual(compare_proposition_multiplicity(before, after), "increased") + + class AdapterCases(unittest.TestCase): def test_reference_binding_passes_five_cases_and_exact_mutations(self) -> None: report = validate_adapter_conformance( @@ -175,29 +234,63 @@ def test_reference_binding_passes_five_cases_and_exact_mutations(self) -> None: self.assertTrue(result.positive_control_passed, result.missing_calls) self.assertTrue(result.mutation_control_passed, result.mutation_failures) - def test_mutation_exception_is_failed_control_not_success(self) -> None: + def test_binding_execution_fault_is_invalid_evidence(self) -> None: class ExplodingBinding(ReferenceConformanceBinding): name = "exploding" def apply_mutation(self, case, importer, adapter, context) -> None: raise RuntimeError("mutation setup broke") - report = validate_adapter_conformance(CORPUS, ROOT, ExplodingBinding) - self.assertFalse(report.passed) - self.assertTrue( - all(not result.mutation_control_passed for result in report.cases) - ) - self.assertTrue( - all( - any("unexpected RuntimeError" in item for item in result.mutation_failures) - for result in report.cases - ) - ) + with self.assertRaisesRegex(ConformanceInputError, "binding execution"): + validate_adapter_conformance(CORPUS, ROOT, ExplodingBinding) + + def test_binding_execution_has_a_hard_timeout(self) -> None: + class SlowBinding(ReferenceConformanceBinding): + def build(self, case): + time.sleep(0.1) + return super().build(case) + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + validate_adapter_conformance(CORPUS, ROOT, SlowBinding) + + +class RuntimeSourceIdentity(unittest.TestCase): + def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None: + report = validate_adapter_conformance(CORPUS, ROOT, ReferenceConformanceBinding) + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, + capture_output=True, text=True, + ).stdout.strip() + tree = subprocess.run( + ["git", "rev-parse", "HEAD^{tree}"], cwd=ROOT, check=True, + capture_output=True, text=True, + ).stdout.strip() + self.assertEqual(report.runtime_commit, commit) + self.assertEqual(report.runtime_tree, tree) + self.assertEqual(report.binding_source["path"], "prototype/conformance.py") + self.assertRegex(report.binding_source["sha256"], r"^[0-9a-f]{64}$") + + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: + marker = ROOT / "conformance-dirty-sentinel.tmp" + marker.write_text("untracked runtime input", encoding="utf-8") + self.addCleanup(marker.unlink, missing_ok=True) + with self.assertRaisesRegex(ConformanceInputError, "dirty"): + validate_adapter_conformance(CORPUS, ROOT, ReferenceConformanceBinding) + + def test_git_timeout_is_invalid_source_evidence(self) -> None: + with patch( + "prototype.conformance.subprocess.run", + side_effect=subprocess.TimeoutExpired(["git"], 5), + ): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + load_corpus(CORPUS, ROOT) class AntiVacuity(unittest.TestCase): def test_all_declared_validator_attacks_are_rejected(self) -> None: - controls = run_validator_anti_vacuity_controls() + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls(corpus, ROOT) self.assertEqual( [item.control_id for item in controls], [ @@ -216,5 +309,32 @@ def test_all_declared_validator_attacks_are_rejected(self) -> None: ], ) + def test_permissive_receipt_validator_makes_empty_receipt_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, receipt_validator=lambda value: (), + ) + self.assertFalse(controls[0].passed) + + def test_no_op_feed_verifier_makes_acceptance_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, feed_verifier=lambda errata, **kwargs: list(errata), + ) + self.assertFalse(controls[1].passed) + + def test_constant_unknown_semantic_runner_makes_diversity_control_fail(self) -> None: + from prototype.semantic import SemanticCoverage + + class ConstantUnknownRunner: + def run(self, probes, config, verifier): + return type("Report", (), {"coverage": SemanticCoverage.UNKNOWN})() + + corpus = load_corpus(CORPUS, ROOT) + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, semantic_runner_factory=ConstantUnknownRunner, + ) + self.assertFalse(controls[2].passed) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index c018c28..e017682 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -25,11 +25,17 @@ def test_version_citation_maturity_security_and_changelog_align(self) -> None: self.assertIn("version: 0.4.0", citation) self.assertIn("date-released: 2026-08-13", citation) self.assertIn("Version 0.4.0", (root / "README.md").read_text()) - self.assertIn("| 0.4.x | Yes |", (root / "SECURITY.md").read_text()) + self.assertIn( + "| 0.4.x | Yes, after `v0.4.0` is published |", + (root / "SECURITY.md").read_text(), + ) changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") self.assertIn("## [0.4.0]", changelog) self.assertIn("Rastislav Drahos", changelog) self.assertIn("2ba1e299b3483b9038d03387345702427608b90b", changelog) + security = (root / "SECURITY.md").read_text(encoding="utf-8") + self.assertIn("beginning with the immutable `v0.4.0` release", security) + self.assertIn("until `v0.4.0` is published", security) class ValidatorPasses(unittest.TestCase): From 143846ec0421cea33e0b431b621fc3fe9e80784d Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:39:48 +0200 Subject: [PATCH 70/88] fix: scope multiplicity to preserved identities --- prototype/conformance.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 81b4712..3afc63d 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -607,6 +607,19 @@ class ReferenceConformanceBinding: """Reference binding; external implementations supply an equivalent class.""" name = "llm-errata-reference" + preserved_proposition_ids = frozenset( + {"fixture:quiet", "fixture:budget", "fixture:pet"} + ) + + @classmethod + def preserved_observations( + cls, adapter: ReferenceConformanceAdapter + ) -> tuple[PropositionObservation, ...]: + return tuple( + item + for item in adapter.proposition_observations() + if item.proposition_id in cls.preserved_proposition_ids + ) def build(self, case: AdapterCase): from prototype.controller import Importer @@ -630,7 +643,7 @@ def build(self, case: AdapterCase): return importer, traced, { "adapter": adapter, "owner": owner, - "before_observations": adapter.proposition_observations(), + "before_observations": self.preserved_observations(adapter), } def erratum(self, case: AdapterCase, context: dict[str, Any]): @@ -686,7 +699,7 @@ def observe( ) triad = dict(receipt.triad) multiplicity = compare_proposition_multiplicity( - context["before_observations"], target.proposition_observations() + context["before_observations"], self.preserved_observations(target) ) preserved = all( target.recall(term) From 59d0f91655c5c15a8040de9a5a1db65e2a304860 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:40:28 +0200 Subject: [PATCH 71/88] fix: retain zero-count proposition identities --- prototype/conformance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 3afc63d..1375318 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -587,16 +587,16 @@ def dispositions(self, root: str) -> dict[str, str]: return result def proposition_observations(self) -> tuple[PropositionObservation, ...]: - counts: dict[str, int] = {} - for identities in self._active_propositions.values(): - for proposition_id in identities: - counts[proposition_id] = counts.get(proposition_id, 0) + 1 labels = { "fixture:diet": "diet", "fixture:quiet": "quiet", "fixture:budget": "budget", "fixture:pet": "pet", } + counts = {proposition_id: 0 for proposition_id in labels} + for identities in self._active_propositions.values(): + for proposition_id in identities: + counts[proposition_id] = counts.get(proposition_id, 0) + 1 return tuple( PropositionObservation(proposition_id, labels[proposition_id], count) for proposition_id, count in sorted(counts.items()) From 0ddef193d68e89390b48dbffdcd0e9550967e3e4 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:41:10 +0200 Subject: [PATCH 72/88] test: require bounded binding construction --- tests/test_conformance.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 6868ac4..9f82ae4 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -254,6 +254,15 @@ def build(self, case): with self.assertRaisesRegex(ConformanceInputError, "timed out"): validate_adapter_conformance(CORPUS, ROOT, SlowBinding) + def test_binding_constructor_has_a_hard_timeout(self) -> None: + class SlowConstructorBinding(ReferenceConformanceBinding): + def __init__(self): + time.sleep(0.1) + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + validate_adapter_conformance(CORPUS, ROOT, SlowConstructorBinding) + class RuntimeSourceIdentity(unittest.TestCase): def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None: From 046dcaa876175c56ffbf824f361091fb2529b7a0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:41:26 +0200 Subject: [PATCH 73/88] fix: bound binding construction time --- prototype/conformance.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 1375318..2862dae 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -914,7 +914,9 @@ class _BindingTimeout(Exception): def _run_with_timeout(action: Callable[[], Any]) -> Any: if threading.current_thread() is not threading.main_thread(): - return action() + raise ConformanceInputError( + "binding execution requires the main thread for timeout enforcement" + ) def expire(signum: int, frame: object) -> None: raise _BindingTimeout @@ -937,7 +939,16 @@ def validate_adapter_conformance( runtime_commit, runtime_tree = _runtime_identity(source_root) corpus = load_corpus(corpus_path, source_root) binding_source = _binding_source(source_root, binding_factory) - binding = binding_factory() + try: + binding = _run_with_timeout(binding_factory) + except _BindingTimeout as error: + raise ConformanceInputError("binding construction timed out") from error + except ConformanceInputError: + raise + except Exception as error: + raise ConformanceInputError( + f"binding construction failed: {type(error).__name__}" + ) from error results = [] for case in corpus.cases: try: From b76814b22a3607a2775ff807c8335d5aa74a1fa3 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:44:55 +0200 Subject: [PATCH 74/88] fix: validate pre-release security transition --- scripts/validate_repo.py | 28 +++++++++++++++++----------- tests/test_validate_repo.py | 7 ++++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 9fc95ec..b928f93 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -593,23 +593,29 @@ def check_document_version_alignment( cells = [cell.strip() for cell in row.split("|")] if len(cells) == 4 and cells[0] == cells[-1] == "": supported_rows.append((cells[1], cells[2])) - supported_yes = [ - version - for version, status in supported_rows - if status == "Yes" - ] previous_minor = int(minor) - 1 - required_unsupported_rows = { + released_rows = { + (f"{major}.{minor}.x", "Yes"), (f"{major}.{previous_minor}.x and earlier", "No"), ("Unreleased development revisions", "No"), } + prepared_rows = { + (f"{major}.{minor}.x", f"Yes, after `v{repository_version}` is published"), + ( + f"{major}.{previous_minor}.x", + f"Yes, until `v{repository_version}` is published", + ), + (f"{major}.{previous_minor - 1}.x and earlier", "No"), + ("Unreleased development revisions", "No"), + } + supported_set = frozenset(supported_rows) reporter.check( "SECURITY supported version", - supported_yes == [f"{major}.{minor}.x"] - and required_unsupported_rows.issubset(set(supported_rows)), - f"SECURITY.md supports {major}.{minor}.x", - "Keep exactly one Yes row for VERSION major.minor.x and No rows for " - "the prior-version family and unreleased revisions.", + supported_set in {frozenset(released_rows), frozenset(prepared_rows)} + and len(supported_rows) == len(supported_set), + f"SECURITY.md is either release-prepared for or supports {major}.{minor}.x", + "Use exactly the conditional pre-release transition rows or the final " + "released VERSION row plus prior/unreleased No rows.", ) diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index e017682..4b0d286 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -385,7 +385,8 @@ def test_security_supported_version_drift_is_rejected(self) -> None: def mutate(root: Path) -> None: current = (root / "VERSION").read_text(encoding="utf-8").strip() major, minor, _ = current.split(".") - rewrite(root / "SECURITY.md", f"| {major}.{minor}.x | Yes |", "| 0.0.x | Yes |") + expected = f"| {major}.{minor}.x | Yes, after `v{current}` is published |" + rewrite(root / "SECURITY.md", expected, "| 0.0.x | Yes |") result = check_after(SCRIPT, mutate) self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) @@ -395,7 +396,7 @@ def test_extra_security_supported_version_is_rejected(self) -> None: def mutate(root: Path) -> None: current = (root / "VERSION").read_text(encoding="utf-8").strip() major, minor, _ = current.split(".") - supported = f"| {major}.{minor}.x | Yes |" + supported = f"| {major}.{minor}.x | Yes, after `v{current}` is published |" rewrite(root / "SECURITY.md", supported, supported + "\n| 0.2.x | Yes |") result = check_after(SCRIPT, mutate) @@ -406,7 +407,7 @@ def test_earlier_security_versions_cannot_be_supported(self) -> None: def mutate(root: Path) -> None: current = (root / "VERSION").read_text(encoding="utf-8").strip() major, minor, _ = current.split(".") - row = f"| {major}.{int(minor) - 1}.x and earlier |" + row = f"| {major}.{int(minor) - 2}.x and earlier |" rewrite( root / "SECURITY.md", f"{row} No |", From d79bfd8db3e293bfe39783e975e789d7d221f8ff Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:55:59 +0200 Subject: [PATCH 75/88] test: isolate dirty-tree conformance fixture --- tests/test_conformance.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 9f82ae4..c4fa213 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -281,11 +281,26 @@ def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None self.assertRegex(report.binding_source["sha256"], r"^[0-9a-f]{64}$") def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: - marker = ROOT / "conformance-dirty-sentinel.tmp" - marker.write_text("untracked runtime input", encoding="utf-8") - self.addCleanup(marker.unlink, missing_ok=True) - with self.assertRaisesRegex(ConformanceInputError, "dirty"): - validate_adapter_conformance(CORPUS, ROOT, ReferenceConformanceBinding) + with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: + source_root = Path(directory) + subprocess.run(["git", "init", "-q"], cwd=source_root, check=True) + marker = source_root / "tracked.txt" + marker.write_text("clean\n", encoding="utf-8") + subprocess.run(["git", "add", "tracked.txt"], cwd=source_root, check=True) + subprocess.run( + [ + "git", "-c", "user.name=LLM Errata Tests", + "-c", "user.email=tests@example.invalid", + "commit", "-q", "-m", "fixture", + ], + cwd=source_root, + check=True, + ) + marker.write_text("dirty\n", encoding="utf-8") + with self.assertRaisesRegex(ConformanceInputError, "dirty"): + validate_adapter_conformance( + CORPUS, source_root, ReferenceConformanceBinding + ) def test_git_timeout_is_invalid_source_evidence(self) -> None: with patch( From 0eff23efaa65efd7e3568e58953c90c6fe4f3241 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:57:36 +0200 Subject: [PATCH 76/88] test: bind receipt and metadata acceptance seams --- tests/test_cli.py | 4 ++++ tests/test_conformance.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index aec115c..d105db2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,6 +15,7 @@ from pathlib import Path from prototype.conformance import ReferenceConformanceBinding +from prototype.receipts import receipt_acceptance_errors REPO_ROOT = Path(__file__).resolve().parents[1] @@ -230,6 +231,9 @@ def test_plan_changes_nothing(self) -> None: class ReceiptsAreVerifiable(CliCase): + def test_empty_receipt_is_rejected_by_shared_acceptance_seam(self) -> None: + self.assertIn("receipt is vacuous", receipt_acceptance_errors({})) + def test_a_genuine_receipt_verifies(self) -> None: self.seed() self.publish_supersession() diff --git a/tests/test_conformance.py b/tests/test_conformance.py index c4fa213..620b286 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -263,6 +263,17 @@ def __init__(self): with self.assertRaisesRegex(ConformanceInputError, "timed out"): validate_adapter_conformance(CORPUS, ROOT, SlowConstructorBinding) + def test_binding_metadata_has_a_hard_timeout(self) -> None: + class SlowMetadataBinding(ReferenceConformanceBinding): + @property + def name(self): + time.sleep(0.1) + return "slow-metadata" + + with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): + with self.assertRaisesRegex(ConformanceInputError, "timed out"): + validate_adapter_conformance(CORPUS, ROOT, SlowMetadataBinding) + class RuntimeSourceIdentity(unittest.TestCase): def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None: @@ -340,6 +351,12 @@ def test_permissive_receipt_validator_makes_empty_receipt_control_fail(self) -> ) self.assertFalse(controls[0].passed) + def test_permissive_production_receipt_schema_makes_control_fail(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + with patch("prototype.receipts.validate_schema", return_value=[]): + controls = run_validator_anti_vacuity_controls(corpus, ROOT) + self.assertFalse(controls[0].passed) + def test_no_op_feed_verifier_makes_acceptance_control_fail(self) -> None: corpus = load_corpus(CORPUS, ROOT) controls = run_validator_anti_vacuity_controls( From 73a2b9dce47fc8e7b2053399c6f72b3f4b68e16e Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Thu, 13 Aug 2026 23:59:35 +0200 Subject: [PATCH 77/88] fix: share receipt acceptance and bound metadata --- prototype/cli.py | 9 ++++++--- prototype/conformance.py | 31 +++++++++++++++++-------------- prototype/receipts.py | 9 +++++++++ tests/test_cli.py | 9 +++++++++ tests/test_conformance.py | 6 ++++-- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index f67fc2b..4a7e674 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -37,7 +37,7 @@ from prototype.controller import Importer, Phase from prototype.errata import Erratum, FeedError, Operation, RootRegistry, read_feed from prototype.lineage import LineageLedger -from prototype.receipts import Receipt +from prototype.receipts import Receipt, receipt_acceptance_errors from prototype.schema import load as load_schema, validate as validate_schema from prototype.semantic import ( RecordedSemanticVerifier, @@ -358,10 +358,13 @@ def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: return EXIT_REFUSED key = ws.importer_verification_key() - schema = load_schema("receipt") bad = 0 for name, payload in receipts: - errors = validate_schema(payload, schema) + errors = receipt_acceptance_errors(payload) + if errors: + bad += 1 + print(f" {name}: signature=not-checked schema={errors[0]} -> BAD") + continue signature = payload.get("signature") rebuilt = Receipt( importer=payload["importer"], diff --git a/prototype/conformance.py b/prototype/conformance.py index 2862dae..9d3f7c7 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -753,17 +753,6 @@ def _run_case( return binding.observe(case, importer, adapter, context, checkpoint, receipt), adapter.calls -def _receipt_errors(value: object) -> tuple[str, ...]: - """Exercise the production receipt schema plus non-vacuity acceptance rule.""" - - from prototype.schema import load as load_schema, validate as validate_schema - - if not isinstance(value, dict) or not value: - return ("receipt is vacuous",) - errors = tuple(validate_schema(value, load_schema("receipt"))) - return errors - - def _anti_vacuity_receipt( corpus: AdapterCorpus, receipt_validator: Callable[[object], tuple[str, ...]] ) -> str: @@ -852,19 +841,21 @@ def run_validator_anti_vacuity_controls( corpus: AdapterCorpus, source_root: Path, *, - receipt_validator: Callable[[object], tuple[str, ...]] = _receipt_errors, + receipt_validator: Callable[[object], tuple[str, ...]] | None = None, feed_verifier: Callable[..., list[object]] | None = None, semantic_runner_factory: Callable[[], object] | None = None, ) -> tuple[ControlResult, ...]: """Install declared flattering mutations against actual acceptance seams.""" from prototype.errata import verify_feed + from prototype.receipts import receipt_acceptance_errors from prototype.semantic import SemanticProbeRunner + receipt = receipt_validator or receipt_acceptance_errors feed = feed_verifier or verify_feed semantic = semantic_runner_factory or SemanticProbeRunner failures = ( - _anti_vacuity_receipt(corpus, receipt_validator), + _anti_vacuity_receipt(corpus, receipt), _anti_vacuity_feed(corpus, feed), _anti_vacuity_semantic(corpus, source_root, semantic), ) @@ -949,6 +940,18 @@ def validate_adapter_conformance( raise ConformanceInputError( f"binding construction failed: {type(error).__name__}" ) from error + try: + binding_name = _run_with_timeout(lambda: binding.name) + except _BindingTimeout as error: + raise ConformanceInputError("binding metadata timed out") from error + except ConformanceInputError: + raise + except Exception as error: + raise ConformanceInputError( + f"binding metadata failed: {type(error).__name__}" + ) from error + if not isinstance(binding_name, str) or not binding_name.strip(): + raise ConformanceInputError("binding metadata name must be non-empty") results = [] for case in corpus.cases: try: @@ -995,7 +998,7 @@ def validate_adapter_conformance( ) ) return ConformanceReport( - binding=binding.name, + binding=binding_name, normative_commit=corpus.normative_target.commit, normative_surface_digest=corpus.normative_target.surface_digest, runtime_commit=runtime_commit, diff --git a/prototype/receipts.py b/prototype/receipts.py index 2f0830d..92644f8 100644 --- a/prototype/receipts.py +++ b/prototype/receipts.py @@ -20,9 +20,18 @@ from typing import Any, Mapping, Sequence from prototype.adapters import Coverage +from prototype.schema import load as load_schema, validate as validate_schema from prototype.signing import VerificationKey +def receipt_acceptance_errors(value: object) -> tuple[str, ...]: + """Return production receipt-acceptance errors, including non-vacuity.""" + + if not isinstance(value, dict) or not value: + return ("receipt is vacuous",) + return tuple(validate_schema(value, load_schema("receipt"))) + + def aggregate_coverage( stores: Mapping[str, Coverage], triad: Mapping[str, str] ) -> Coverage: diff --git a/tests/test_cli.py b/tests/test_cli.py index d105db2..07eb465 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -240,6 +240,15 @@ def test_a_genuine_receipt_verifies(self) -> None: self.quarantine_and_repair() self.assertEqual(self.run_cli("verify").returncode, EXIT_OK) + def test_empty_receipt_is_refused_without_traceback(self) -> None: + self.seed() + path = self.cwd / ".errata" / "receipts" / "empty.json" + path.write_text("{}", encoding="utf-8") + result = self.run_cli("verify") + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertIn("receipt is vacuous", result.stdout) + self.assertNotIn("Traceback", result.stderr) + def test_a_tampered_aggregate_is_caught(self) -> None: self.seed() self.publish_supersession() diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 620b286..dfea682 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -271,7 +271,9 @@ def name(self): return "slow-metadata" with patch("prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01): - with self.assertRaisesRegex(ConformanceInputError, "timed out"): + with self.assertRaisesRegex( + ConformanceInputError, "binding metadata timed out" + ): validate_adapter_conformance(CORPUS, ROOT, SlowMetadataBinding) @@ -353,7 +355,7 @@ def test_permissive_receipt_validator_makes_empty_receipt_control_fail(self) -> def test_permissive_production_receipt_schema_makes_control_fail(self) -> None: corpus = load_corpus(CORPUS, ROOT) - with patch("prototype.receipts.validate_schema", return_value=[]): + with patch("prototype.receipts.receipt_acceptance_errors", return_value=()): controls = run_validator_anti_vacuity_controls(corpus, ROOT) self.assertFalse(controls[0].passed) From 120cda881cf2f76169d28feb7228429146103d09 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:06:47 +0200 Subject: [PATCH 78/88] test: reject malformed receipt JSON safely --- tests/test_cli.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 07eb465..fe18e95 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -249,6 +249,15 @@ def test_empty_receipt_is_refused_without_traceback(self) -> None: self.assertIn("receipt is vacuous", result.stdout) self.assertNotIn("Traceback", result.stderr) + def test_malformed_receipt_json_is_refused_without_traceback(self) -> None: + self.seed() + path = self.cwd / ".errata" / "receipts" / "malformed.json" + path.write_text("{not json", encoding="utf-8") + result = self.run_cli("verify") + self.assertEqual(result.returncode, EXIT_REFUSED) + self.assertIn("receipt JSON is unreadable", result.stdout) + self.assertNotIn("Traceback", result.stderr) + def test_a_tampered_aggregate_is_caught(self) -> None: self.seed() self.publish_supersession() From 3d5dc890e47977f45500f689fc9667b052f14936 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:07:33 +0200 Subject: [PATCH 79/88] fix: refuse unreadable receipt evidence --- prototype/cli.py | 8 ++++++-- prototype/workspace.py | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index 4a7e674..b22293e 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -49,7 +49,7 @@ ) from prototype.signing import Ed25519Signer from prototype.sqlite_store import SqliteAdapter -from prototype.workspace import Workspace +from prototype.workspace import ReceiptReadError, Workspace EXIT_OK = 0 @@ -352,7 +352,11 @@ def cmd_audit(ws: Workspace, args: argparse.Namespace) -> int: def cmd_verify(ws: Workspace, args: argparse.Namespace) -> int: """Check every receipt against the published key and the published schema.""" - receipts = ws.all_receipts() + try: + receipts = ws.all_receipts() + except ReceiptReadError as error: + print(f" {error} -> BAD") + return EXIT_REFUSED if not receipts: print("no receipts to verify", file=sys.stderr) return EXIT_REFUSED diff --git a/prototype/workspace.py b/prototype/workspace.py index c247065..9814982 100644 --- a/prototype/workspace.py +++ b/prototype/workspace.py @@ -25,6 +25,10 @@ from prototype.signing import Ed25519Signer, VerificationKey +class ReceiptReadError(ValueError): + """A persisted receipt cannot be parsed as JSON evidence.""" + + class Workspace: def __init__(self, root: Path) -> None: self.root = Path(root) @@ -238,10 +242,16 @@ def _atomic_json(self, path: Path, payload: object) -> None: def all_receipts(self) -> list[tuple[str, dict[str, Any]]]: if not self.receipts_dir.is_dir(): return [] - return [ - (p.name, json.loads(p.read_text(encoding="utf-8"))) - for p in sorted(self.receipts_dir.glob("*.json")) - ] + receipts = [] + for path in sorted(self.receipts_dir.glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ReceiptReadError( + f"{path.name}: receipt JSON is unreadable" + ) from error + receipts.append((path.name, payload)) + return receipts def latest_receipt(self) -> dict[str, Any] | None: receipts = self.all_receipts() From 4ab1a0fe4046f3717c8fbf86ea04ae817807a1d6 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:15:00 +0200 Subject: [PATCH 80/88] ci: fetch immutable conformance history --- .github/workflows/validate.yml | 4 ++++ scripts/validate_repo.py | 10 ++++++++++ tests/test_validate_repo.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index f8abdd1..c4c95c6 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -24,6 +24,10 @@ jobs: steps: - name: Check out the repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1, Node 24 + with: + # Conformance binds historical immutable normative commits. A shallow + # checkout cannot verify those Git objects and must not run the suite. + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0, Node 24 diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index b928f93..7a6fd5d 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -394,6 +394,16 @@ def check_github_actions_runtime(reporter: Reporter) -> None: "Restore immutable Node 24 action pins. " + "; ".join(violations), ) + validate_workflow = ROOT / ".github" / "workflows" / "validate.yml" + validate_text = read_utf8(validate_workflow) if validate_workflow.is_file() else "" + reporter.check( + "GitHub Actions source history", + validate_text.count("fetch-depth: 0") == 1, + "validation checkout fetches immutable Git history required by conformance", + "Set actions/checkout fetch-depth to 0 in validate.yml so historical " + "normative commits are available to fail-closed conformance checks.", + ) + def check_publication_metadata(reporter: Reporter) -> None: readme_path = ROOT / "README.md" diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 4b0d286..12af69d 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -479,6 +479,12 @@ def test_workflows_pin_node24_action_releases(self) -> None: self.assertIn("actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", workflows) self.assertIn("actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", workflows) + def test_validate_workflow_fetches_immutable_history_for_conformance(self) -> None: + workflow = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "validate.yml" + ).read_text(encoding="utf-8") + self.assertIn("fetch-depth: 0", workflow) + def test_validator_rejects_deprecated_action_major(self) -> None: def mutate(root: Path) -> None: path = root / ".github" / "workflows" / "links.yml" @@ -496,6 +502,14 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("GitHub Actions Node 24 pins", result.stdout) + def test_validator_rejects_shallow_validate_checkout(self) -> None: + def mutate(root: Path) -> None: + rewrite(root / ".github" / "workflows" / "validate.yml", "fetch-depth: 0", "fetch-depth: 1") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("GitHub Actions source history", result.stdout) + if __name__ == "__main__": unittest.main() From 1fa500c4fdbafd4098644ee00d9b9378c474bf81 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:18:22 +0200 Subject: [PATCH 81/88] test: bind history guard to checkout step --- scripts/validate_repo.py | 10 +++++++++- tests/test_validate_repo.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/validate_repo.py b/scripts/validate_repo.py index 7a6fd5d..db53930 100644 --- a/scripts/validate_repo.py +++ b/scripts/validate_repo.py @@ -396,9 +396,17 @@ def check_github_actions_runtime(reporter: Reporter) -> None: validate_workflow = ROOT / ".github" / "workflows" / "validate.yml" validate_text = read_utf8(validate_workflow) if validate_workflow.is_file() else "" + checkout_step = re.search( + rf"(?ms)^\s{{6}}- name: Check out the repository\s*$\n" + rf"^\s{{8}}uses: actions/checkout@{GITHUB_ACTION_PINS['actions/checkout']}[^\n]*$\n" + rf"^\s{{8}}with:\s*$\n" + rf"(?:(?:^\s{{10}}#.*$\n)*)" + rf"^\s{{10}}fetch-depth:\s*0\s*$", + validate_text, + ) reporter.check( "GitHub Actions source history", - validate_text.count("fetch-depth: 0") == 1, + checkout_step is not None, "validation checkout fetches immutable Git history required by conformance", "Set actions/checkout fetch-depth to 0 in validate.yml so historical " "normative commits are available to fail-closed conformance checks.", diff --git a/tests/test_validate_repo.py b/tests/test_validate_repo.py index 12af69d..73376ab 100644 --- a/tests/test_validate_repo.py +++ b/tests/test_validate_repo.py @@ -510,6 +510,18 @@ def mutate(root: Path) -> None: self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) self.assertIn("GitHub Actions source history", result.stdout) + def test_history_guard_ignores_comment_camouflage(self) -> None: + def mutate(root: Path) -> None: + path = root / ".github" / "workflows" / "validate.yml" + text = path.read_text(encoding="utf-8") + text = text.replace("fetch-depth: 0", "fetch-depth: 1", 1) + text += "\n# legacy requirement text: fetch-depth: 0\n" + path.write_text(text, encoding="utf-8") + + result = check_after(SCRIPT, mutate) + self.assertEqual(result.returncode, EXIT_FAIL, result.stdout) + self.assertIn("GitHub Actions source history", result.stdout) + if __name__ == "__main__": unittest.main() From 1aa9db88dd442623d172b31de622cfbcc73e5c53 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:32:15 +0200 Subject: [PATCH 82/88] fix: close conformance review findings --- prototype/cli.py | 6 ++- prototype/conformance.py | 75 ++++++++++++++++++++++++++-------- scripts/check_readiness.py | 13 ++++++ spec/README.md | 5 +++ tests/test_cli.py | 7 ++++ tests/test_conformance.py | 83 ++++++++++++++++++++++++++++++++++++++ tests/test_readiness.py | 1 + 7 files changed, 173 insertions(+), 17 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index b22293e..81c851e 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -77,7 +77,10 @@ def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: try: report = validate_adapter_conformance( - args.corpus, args.source_root, _binding_factory(args.binding) + args.corpus, + args.source_root, + _binding_factory(args.binding), + binding_root=args.binding_root, ) except ConformanceInputError as error: print(f"invalid conformance evidence: {error}", file=sys.stderr) @@ -476,6 +479,7 @@ def build_parser() -> argparse.ArgumentParser: ) adapter_conformance.add_argument("--corpus", required=True, type=Path) adapter_conformance.add_argument("--source-root", required=True, type=Path) + adapter_conformance.add_argument("--binding-root", type=Path) adapter_conformance.add_argument("--binding") return parser diff --git a/prototype/conformance.py b/prototype/conformance.py index 9d3f7c7..92cec66 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -29,6 +29,11 @@ def __init__(self, target: object) -> None: def calls(self) -> tuple[str, ...]: return tuple(object.__getattribute__(self, "_calls")) + def reset_calls(self) -> None: + """Start a new trace window around controller-issued lifecycle calls.""" + + object.__getattribute__(self, "_calls").clear() + def __getattr__(self, name: str) -> Any: value = getattr(self.target, name) if not callable(value): @@ -64,7 +69,7 @@ def compare(want: object, got: object, path: str) -> None: child = f"{path}.{key}" if path else key failures.append(f"{child}: unexpected") return - if want != got: + if type(want) is not type(got) or want != got: failures.append(f"{path}: expected {want!r}, got {got!r}") compare(expected, observed, "") @@ -131,6 +136,7 @@ class AdapterCorpus: validator_controls: tuple[dict[str, str], ...] status: str evidence_boundary: str + sha256: str @dataclass(frozen=True) @@ -191,6 +197,7 @@ class ConformanceReport: binding: str normative_commit: str normative_surface_digest: str + corpus_sha256: str runtime_commit: str | None runtime_tree: str binding_source: dict[str, str] @@ -214,6 +221,7 @@ def to_dict(self) -> dict[str, Any]: "commit": self.normative_commit, "surface_digest": self.normative_surface_digest, }, + "corpus_sha256": self.corpus_sha256, "runtime_commit": self.runtime_commit, "runtime_tree": self.runtime_tree, "binding_source": dict(sorted(self.binding_source.items())), @@ -356,9 +364,16 @@ def _validate_outcome(value: object, operation: str, label: str) -> dict[str, An return outcome -def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: +def load_corpus( + path: Path, source_root: Path, *, require_canonical_path: bool = False +) -> AdapterCorpus: + if require_canonical_path and path.resolve() != ( + source_root / "spec" / "adapter-conformance.json" + ).resolve(): + raise ConformanceInputError("corpus is not the canonical checked-in corpus") try: - payload = json.loads(path.read_text(encoding="utf-8")) + corpus_bytes = path.read_bytes() + payload = json.loads(corpus_bytes.decode("utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: raise ConformanceInputError("corpus is not readable canonical JSON") from error root = _exact(payload, ROOT_KEYS, "corpus") @@ -392,6 +407,12 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: raise ConformanceInputError(f"{case_id} operation is invalid") normative = _exact(case["normative"], NORMATIVE_KEYS, f"{case_id} normative") commit = _nonempty(normative["commit"], f"{case_id} normative commit") + try: + _git(source_root, "merge-base", "--is-ancestor", commit, "HEAD") + except ConformanceInputError as error: + raise ConformanceInputError( + f"{case_id} normative commit is not reachable from runtime history" + ) from error relative = _nonempty(normative["path"], f"{case_id} normative path") quote = _nonempty(normative["quote"], f"{case_id} normative quotation") source = _git(source_root, "show", f"{commit}:{relative}").decode("utf-8") @@ -429,6 +450,7 @@ def load_corpus(path: Path, source_root: Path) -> AdapterCorpus: validator_controls=tuple(controls), status=_nonempty(root["status"], "status"), evidence_boundary=_nonempty(root["evidence_boundary"], "evidence boundary"), + sha256=hashlib.sha256(corpus_bytes).hexdigest(), ) @@ -748,9 +770,12 @@ def _run_case( if mutate: binding.apply_mutation(case, importer, adapter, context) erratum = binding.erratum(case, context) + adapter.reset_calls() checkpoint = importer.quarantine(erratum) receipt = importer.repair_quarantined(erratum, checkpoint) - return binding.observe(case, importer, adapter, context, checkpoint, receipt), adapter.calls + calls = adapter.calls + observed = binding.observe(case, importer, adapter, context, checkpoint, receipt) + return observed, calls def _anti_vacuity_receipt( @@ -766,7 +791,7 @@ def _anti_vacuity_receipt( def _anti_vacuity_feed( corpus: AdapterCorpus, feed_verifier: Callable[..., list[object]] ) -> str: - from prototype.errata import Erratum, Operation, RootRegistry + from prototype.errata import Erratum, FeedError, Operation, RootRegistry from prototype.signing import Ed25519Signer mutation = next( @@ -791,9 +816,15 @@ def _anti_vacuity_feed( ) try: feed_verifier([event], owner=owner.public, roots=RootRegistry({"fact:diet"})) - except Exception as error: - message = str(error) - return mutation["required_failure"] if "gap" in message else "" + except FeedError as error: + expected = ( + "anti-vacuity-gap: gap at sequence 2, expected 1. " + "A missing erratum may be the one that retired the state this importer " + "is about to serve." + ) + return mutation["required_failure"] if str(error) == expected else "" + except Exception: + return "" return "" @@ -879,24 +910,34 @@ def _runtime_identity(root: Path) -> tuple[str, str]: def _binding_source( - source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding] + binding_factory: Callable[[], ReferenceConformanceBinding], + binding_root: Path | None, ) -> dict[str, str]: try: file_path = Path(inspect.getsourcefile(binding_factory) or "").resolve() - relative = file_path.relative_to(source_root.resolve()).as_posix() + root = binding_root.resolve() if binding_root is not None else Path( + _git(file_path.parent, "rev-parse", "--show-toplevel").decode().strip() + ).resolve() + relative = file_path.relative_to(root).as_posix() payload = file_path.read_bytes() - except (OSError, ValueError) as error: + except (OSError, TypeError, ValueError) as error: raise ConformanceInputError( - "binding source is not inside the declared source root" + "binding source is not inside a clean Git repository" ) from error if relative.startswith(".git/"): raise ConformanceInputError("binding source is not executable repository source") tracked = set( - _git(source_root, "ls-files", "--cached").decode("utf-8").splitlines() + _git(root, "ls-files", "--cached").decode("utf-8").splitlines() ) if relative not in tracked: raise ConformanceInputError("binding source is not tracked by the runtime tree") - return {"path": relative, "sha256": hashlib.sha256(payload).hexdigest()} + commit, tree = _runtime_identity(root) + return { + "commit": commit, + "tree": tree, + "path": relative, + "sha256": hashlib.sha256(payload).hexdigest(), + } class _BindingTimeout(Exception): @@ -926,10 +967,11 @@ def validate_adapter_conformance( corpus_path: Path, source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding], + binding_root: Path | None = None, ) -> ConformanceReport: runtime_commit, runtime_tree = _runtime_identity(source_root) - corpus = load_corpus(corpus_path, source_root) - binding_source = _binding_source(source_root, binding_factory) + corpus = load_corpus(corpus_path, source_root, require_canonical_path=True) + binding_source = _binding_source(binding_factory, binding_root) try: binding = _run_with_timeout(binding_factory) except _BindingTimeout as error: @@ -1001,6 +1043,7 @@ def validate_adapter_conformance( binding=binding_name, normative_commit=corpus.normative_target.commit, normative_surface_digest=corpus.normative_target.surface_digest, + corpus_sha256=corpus.sha256, runtime_commit=runtime_commit, runtime_tree=runtime_tree, binding_source=binding_source, diff --git a/scripts/check_readiness.py b/scripts/check_readiness.py index 2cfb77d..0464da0 100644 --- a/scripts/check_readiness.py +++ b/scripts/check_readiness.py @@ -188,6 +188,7 @@ def g2_surface_files(root: Path = ROOT) -> tuple[str, ...]: tuple(sorted((root / "prototype").glob("*.py"))), (root / "prototype" / "README.md",), (root / "spec" / "README.md",), + (root / "spec" / "adapter-conformance.json",), tuple(sorted((root / "spec").glob("*.schema.json"))), tuple(sorted((root / "spec" / "vectors").glob("*.json"))), tuple(sorted((root / "spec" / "semantic").glob("*.json"))), @@ -222,8 +223,20 @@ def g2_surface_digest(root: Path = ROOT) -> str: def g2_surface_digest_at_commit(commit: str, root: Path = ROOT) -> str: if not reviewed_commit_exists(commit, root): raise OSError("reviewed commit is unavailable") + listed = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", commit], + cwd=root, + capture_output=True, + check=False, + text=True, + ) + if listed.returncode != 0: + raise OSError("reviewed commit tree is unavailable") + commit_files = set(listed.stdout.splitlines()) entries = [] for relative in g2_surface_files(root): + if relative == "spec/adapter-conformance.json" and relative not in commit_files: + continue result = subprocess.run( ["git", "show", f"{commit}:{relative}"], cwd=root, capture_output=True, check=False ) diff --git a/spec/README.md b/spec/README.md index 070fad0..b8afb96 100644 --- a/spec/README.md +++ b/spec/README.md @@ -41,6 +41,11 @@ python3 -m prototype.cli adapter-conformance \ --source-root . ``` +For an independently implemented binding in another checkout, add +`--binding module:factory --binding-root /path/to/its/clean/git/checkout`. +Reports bind the LLM Errata runtime commit/tree, exact corpus SHA-256, and the +binding repository commit/tree plus tracked source path and SHA-256 separately. + Exit `0` means the supplied binding and validator controls passed this internal corpus. Exit `1` means a behavioral or mutation control failed. Exit `2` means source, corpus, binding, or execution evidence is invalid or diff --git a/tests/test_cli.py b/tests/test_cli.py index fe18e95..e2db04d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -90,6 +90,7 @@ def test_reference_binding_emits_canonical_passing_report(self) -> None: self.assertTrue(payload["passed"]) self.assertEqual(len(payload["cases"]), 5) self.assertEqual(len(payload["validator_controls"]), 3) + self.assertRegex(payload["corpus_sha256"], r"^[0-9a-f]{64}$") self.assertIn("not G2 or G4 evidence", payload["evidence_boundary"]) self.assertEqual(result.stdout.strip(), json.dumps( payload, sort_keys=True, separators=(",", ":") @@ -112,6 +113,12 @@ def test_invalid_source_evidence_exits_two(self) -> None: self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) self.assertIn("invalid conformance evidence", result.stderr) + def test_sourceless_binding_factory_exits_two_without_traceback(self) -> None: + result = self.run_conformance("--binding", "builtins:dict") + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertEqual(result.stdout, "") + self.assertNotIn("Traceback", result.stderr) + class WorkspaceLifecycle(CliCase): def test_commands_refuse_before_init(self) -> None: diff --git a/tests/test_conformance.py b/tests/test_conformance.py index dfea682..b200cc9 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -59,6 +59,20 @@ def test_wrong_surface_digest_is_refused(self) -> None: with self.assertRaisesRegex(ConformanceInputError, "surface digest"): load_corpus(path, ROOT) + def test_noncanonical_corpus_path_is_refused_for_execution(self) -> None: + path = self.changed_corpus(lambda value: None) + with self.assertRaisesRegex(ConformanceInputError, "canonical checked-in"): + load_corpus(path, ROOT, require_canonical_path=True) + + def test_normative_source_must_be_reachable_from_runtime_history(self) -> None: + path = self.changed_corpus( + lambda value: value["cases"][0]["normative"].__setitem__( + "commit", "f" * 40 + ) + ) + with self.assertRaisesRegex(ConformanceInputError, "reachable"): + load_corpus(path, ROOT) + def test_new_current_surface_files_do_not_change_historical_manifest(self) -> None: corpus = load_corpus(CORPUS, ROOT) self.assertEqual( @@ -139,6 +153,20 @@ class Adapter: self.assertEqual(traced.name, "target") self.assertEqual(traced.calls, ()) + def test_trace_window_can_exclude_fixture_and_observation_calls(self) -> None: + class Adapter: + def coverage(self, root: str) -> str: + return root + + traced = TracingAdapter(Adapter()) + traced.coverage("fixture") + traced.reset_calls() + traced.coverage("controller") + calls = traced.calls + traced.coverage("observation") + self.assertEqual(calls, ("coverage",)) + self.assertEqual(traced.calls, ("coverage", "coverage")) + class CompleteComparison(unittest.TestCase): def outcome(self) -> dict[str, object]: @@ -189,6 +217,17 @@ def test_missing_or_extra_fields_are_rejected(self) -> None: self.assertIn("receipt.names_store: missing", failures) self.assertIn("store.unexpected: unexpected", failures) + def test_json_scalar_types_must_match_exactly(self) -> None: + for wrong in (1, 1.0): + with self.subTest(wrong=wrong): + expected = self.outcome() + observed = self.outcome() + observed["receipt"]["names_store"] = wrong + self.assertIn( + f"receipt.names_store: expected True, got {wrong!r}", + compare_complete_outcome(expected, observed), + ) + class PropositionMultiplicity(unittest.TestCase): def test_reference_observations_use_stable_provider_local_ids_and_counts(self) -> None: @@ -292,6 +331,39 @@ def test_report_binds_clean_runtime_commit_tree_and_binding_source(self) -> None self.assertEqual(report.runtime_tree, tree) self.assertEqual(report.binding_source["path"], "prototype/conformance.py") self.assertRegex(report.binding_source["sha256"], r"^[0-9a-f]{64}$") + self.assertEqual(report.binding_source["commit"], commit) + self.assertEqual(report.binding_source["tree"], tree) + self.assertRegex(report.corpus_sha256, r"^[0-9a-f]{64}$") + + def test_external_binding_can_be_bound_to_its_own_clean_repository(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-") as directory: + binding_root = Path(directory) + source = binding_root / "binding.py" + source.write_text("def factory():\n return None\n", encoding="utf-8") + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "binding.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=binding_root, check=True) + namespace: dict[str, object] = {} + exec(compile(source.read_text(), str(source), "exec"), namespace) + factory = namespace["factory"] + factory.__module__ = "binding" + with patch("prototype.conformance.inspect.getsourcefile", return_value=str(source)): + from prototype.conformance import _binding_source + + identity = _binding_source(factory, binding_root) + self.assertEqual(identity["path"], "binding.py") + self.assertRegex(identity["commit"], r"^[0-9a-f]{40}$") + + def test_sourceless_binding_factory_is_invalid_evidence(self) -> None: + from prototype.conformance import _binding_source + + with self.assertRaisesRegex(ConformanceInputError, "clean Git repository"): + _binding_source(dict, None) def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: @@ -366,6 +438,17 @@ def test_no_op_feed_verifier_makes_acceptance_control_fail(self) -> None: ) self.assertFalse(controls[1].passed) + def test_unrelated_gap_exception_cannot_satisfy_feed_control(self) -> None: + corpus = load_corpus(CORPUS, ROOT) + + def crash(*args, **kwargs): + raise RuntimeError("unrelated gap in logging") + + controls = run_validator_anti_vacuity_controls( + corpus, ROOT, feed_verifier=crash, + ) + self.assertFalse(controls[1].passed) + def test_constant_unknown_semantic_runner_makes_diversity_control_fail(self) -> None: from prototype.semantic import SemanticCoverage diff --git a/tests/test_readiness.py b/tests/test_readiness.py index d2c09ff..20177e5 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -89,6 +89,7 @@ def test_g2_surface_includes_checkpoint_contract_and_tests(self) -> None: files = set(g2_surface_files()) self.assertIn("prototype/checkpoints.py", files) self.assertIn("tests/test_checkpoints.py", files) + self.assertIn("spec/adapter-conformance.json", files) def test_g6_complete_measured_report_is_commit_and_deployment_bound(self) -> None: with repo_copy() as source, tempfile.TemporaryDirectory() as temp: From 03d1a23877eb7b5a43bbdd55fac0a9fca7f871be Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:43:05 +0200 Subject: [PATCH 83/88] fix: bound external binding imports --- prototype/cli.py | 29 +++++-------- prototype/conformance.py | 86 ++++++++++++++++++++++++++++++++++++++- tests/test_cli.py | 36 ++++++++++++++-- tests/test_conformance.py | 41 +++++++++++++++++++ 4 files changed, 169 insertions(+), 23 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index 81c851e..ded61e4 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -21,7 +21,6 @@ from __future__ import annotations import argparse -import importlib import json import sys from datetime import datetime, timezone @@ -32,6 +31,7 @@ from prototype.conformance import ( ConformanceInputError, ReferenceConformanceBinding, + load_binding_factory, validate_adapter_conformance, ) from prototype.controller import Importer, Phase @@ -57,30 +57,23 @@ EXIT_INCONCLUSIVE = 2 -def _binding_factory(value: str | None): - if value is None: - return ReferenceConformanceBinding - try: - module_name, object_name = value.split(":", 1) - factory = getattr(importlib.import_module(module_name), object_name) - except (AttributeError, ImportError, ValueError) as error: - raise ConformanceInputError( - "binding must be an importable module:factory" - ) from error - if not callable(factory): - raise ConformanceInputError("binding factory must be callable") - return factory - - def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: """Run provider-neutral adapter cases and validator self-controls.""" try: + if args.binding is None: + factory = ReferenceConformanceBinding + source = None + binding_root = args.binding_root + else: + binding_root = args.binding_root or args.source_root + factory, source = load_binding_factory(args.binding, binding_root) report = validate_adapter_conformance( args.corpus, args.source_root, - _binding_factory(args.binding), - binding_root=args.binding_root, + factory, + binding_root=binding_root, + binding_source=source, ) except ConformanceInputError as error: print(f"invalid conformance evidence: {error}", file=sys.stderr) diff --git a/prototype/conformance.py b/prototype/conformance.py index 92cec66..633504c 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -3,11 +3,13 @@ from __future__ import annotations import hashlib +import importlib import inspect import json import re import signal import subprocess +import sys import threading from dataclasses import dataclass from pathlib import Path @@ -940,6 +942,85 @@ def _binding_source( } +def load_binding_factory( + value: str, + binding_root: Path, +) -> tuple[Callable[[], ReferenceConformanceBinding], dict[str, str]]: + """Admit tracked module bytes before bounded import executes them.""" + + try: + module_name, object_name = value.split(":", 1) + except ValueError as error: + raise ConformanceInputError( + "binding must be an importable module:factory" + ) from error + identifier = re.compile(r"^[A-Za-z_]\w*$") + parts = module_name.split(".") + if not parts or not all(identifier.fullmatch(part) for part in parts) or not identifier.fullmatch( + object_name + ): + raise ConformanceInputError("binding must be an importable module:factory") + + root = binding_root.resolve() + module_path = root.joinpath(*parts).with_suffix(".py") + package_path = root.joinpath(*parts, "__init__.py") + candidates = [path for path in (module_path, package_path) if path.is_file()] + if len(candidates) != 1: + raise ConformanceInputError("binding module must resolve to one repository file") + admitted_path = candidates[0].resolve() + relative = admitted_path.relative_to(root).as_posix() + tracked = set(_git(root, "ls-files", "--cached").decode("utf-8").splitlines()) + if relative not in tracked: + raise ConformanceInputError("binding source is not tracked by the runtime tree") + commit, tree = _runtime_identity(root) + source = { + "commit": commit, + "tree": tree, + "path": relative, + "sha256": hashlib.sha256(admitted_path.read_bytes()).hexdigest(), + } + + try: + search_locations = [str(admitted_path.parent)] if admitted_path.name == "__init__.py" else None + spec = importlib.util.spec_from_file_location( + module_name, admitted_path, submodule_search_locations=search_locations + ) + if spec is None or spec.loader is None: + raise ImportError("binding module loader is unavailable") + module = importlib.util.module_from_spec(spec) + prior_module = sys.modules.get(module_name) + sys.modules[module_name] = module + + def execute_import() -> None: + spec.loader.exec_module(module) + + _run_with_timeout(execute_import) + factory = getattr(module, object_name) + except _BindingTimeout as error: + raise ConformanceInputError("binding import timed out") from error + except BaseException as error: + if isinstance(error, KeyboardInterrupt): + raise + raise ConformanceInputError( + f"binding import failed: {type(error).__name__}" + ) from error + finally: + if "prior_module" in locals(): + if prior_module is None: + sys.modules.pop(module_name, None) + else: + sys.modules[module_name] = prior_module + if not callable(factory): + raise ConformanceInputError("binding factory must be callable") + try: + loaded_path = Path(inspect.getsourcefile(factory) or "").resolve() + except (OSError, TypeError, ValueError) as error: + raise ConformanceInputError("binding factory has no admitted source") from error + if loaded_path != admitted_path: + raise ConformanceInputError("binding factory source differs from admitted module") + return factory, source + + class _BindingTimeout(Exception): pass @@ -968,10 +1049,11 @@ def validate_adapter_conformance( source_root: Path, binding_factory: Callable[[], ReferenceConformanceBinding], binding_root: Path | None = None, + binding_source: dict[str, str] | None = None, ) -> ConformanceReport: runtime_commit, runtime_tree = _runtime_identity(source_root) corpus = load_corpus(corpus_path, source_root, require_canonical_path=True) - binding_source = _binding_source(binding_factory, binding_root) + source_identity = binding_source or _binding_source(binding_factory, binding_root) try: binding = _run_with_timeout(binding_factory) except _BindingTimeout as error: @@ -1046,7 +1128,7 @@ def validate_adapter_conformance( corpus_sha256=corpus.sha256, runtime_commit=runtime_commit, runtime_tree=runtime_tree, - binding_source=binding_source, + binding_source=source_identity, cases=tuple(results), validator_controls=run_validator_anti_vacuity_controls(corpus, source_root), provenance=corpus.provenance, diff --git a/tests/test_cli.py b/tests/test_cli.py index e2db04d..87d62e4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,14 +41,19 @@ def setUp(self) -> None: def tearDown(self) -> None: self._tmp.cleanup() - def run_cli(self, *args: str) -> subprocess.CompletedProcess[str]: + def run_cli( + self, *args: str, extra_pythonpath: Path | None = None + ) -> subprocess.CompletedProcess[str]: + pythonpath = str(REPO_ROOT) + if extra_pythonpath is not None: + pythonpath = f"{extra_pythonpath}:{pythonpath}" return subprocess.run( [sys.executable, "-m", "prototype.cli", "--workspace", ".errata", *args], cwd=self.cwd, capture_output=True, text=True, check=False, - env={"PYTHONPATH": str(REPO_ROOT), "PATH": "/usr/bin:/bin"}, + env={"PYTHONPATH": pythonpath, "PATH": "/usr/bin:/bin"}, ) def seed(self) -> None: @@ -75,12 +80,15 @@ def quarantine_and_repair(self) -> subprocess.CompletedProcess[str]: class AdapterConformanceCommand(CliCase): - def run_conformance(self, *extra: str) -> subprocess.CompletedProcess[str]: + def run_conformance( + self, *extra: str, extra_pythonpath: Path | None = None + ) -> subprocess.CompletedProcess[str]: return self.run_cli( "adapter-conformance", "--corpus", str(REPO_ROOT / "spec" / "adapter-conformance.json"), "--source-root", str(REPO_ROOT), *extra, + extra_pythonpath=extra_pythonpath, ) def test_reference_binding_emits_canonical_passing_report(self) -> None: @@ -119,6 +127,28 @@ def test_sourceless_binding_factory_exits_two_without_traceback(self) -> None: self.assertEqual(result.stdout, "") self.assertNotIn("Traceback", result.stderr) + def test_crashing_binding_import_exits_two_without_traceback(self) -> None: + binding_root = self.cwd / "binding" + binding_root.mkdir() + (binding_root / "broken.py").write_text( + 'raise RuntimeError("top-level failure")\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "CLI Tests"), + ("git", "add", "broken.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=binding_root, check=True) + result = self.run_conformance( + "--binding", "broken:factory", "--binding-root", str(binding_root), + extra_pythonpath=binding_root, + ) + self.assertEqual(result.returncode, EXIT_INCONCLUSIVE) + self.assertIn("binding import failed: RuntimeError", result.stderr) + self.assertNotIn("Traceback", result.stderr) + class WorkspaceLifecycle(CliCase): def test_commands_refuse_before_init(self) -> None: diff --git a/tests/test_conformance.py b/tests/test_conformance.py index b200cc9..597bae8 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -8,6 +8,7 @@ import json import subprocess +import sys import tempfile import time import unittest @@ -23,6 +24,7 @@ compare_complete_outcome, compare_proposition_multiplicity, load_corpus, + load_binding_factory, run_validator_anti_vacuity_controls, validate_adapter_conformance, ) @@ -365,6 +367,45 @@ def test_sourceless_binding_factory_is_invalid_evidence(self) -> None: with self.assertRaisesRegex(ConformanceInputError, "clean Git repository"): _binding_source(dict, None) + def test_binding_import_runtime_error_is_invalid_evidence(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + (root / "broken.py").write_text( + 'raise RuntimeError("top-level failure")\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "broken.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + with patch("sys.path", [str(root), *sys.path]): + with self.assertRaisesRegex(ConformanceInputError, "RuntimeError"): + load_binding_factory("broken:factory", root) + + def test_binding_import_has_a_hard_timeout(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + (root / "slow.py").write_text( + "import time\ntime.sleep(0.1)\ndef factory():\n return None\n", + encoding="utf-8", + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "slow.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + with patch("sys.path", [str(root), *sys.path]), patch( + "prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01 + ): + with self.assertRaisesRegex(ConformanceInputError, "import timed out"): + load_binding_factory("slow:factory", root) + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From 41fb070cd8baccd2db8a6aaacabd4d6886e66bfe Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:47:47 +0200 Subject: [PATCH 84/88] fix: execute admitted binding bytes --- prototype/conformance.py | 21 +++++++++++++++------ tests/test_conformance.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 633504c..da3a596 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -973,26 +973,32 @@ def load_binding_factory( if relative not in tracked: raise ConformanceInputError("binding source is not tracked by the runtime tree") commit, tree = _runtime_identity(root) + admitted_bytes = admitted_path.read_bytes() source = { "commit": commit, "tree": tree, "path": relative, - "sha256": hashlib.sha256(admitted_path.read_bytes()).hexdigest(), + "sha256": hashlib.sha256(admitted_bytes).hexdigest(), } try: search_locations = [str(admitted_path.parent)] if admitted_path.name == "__init__.py" else None - spec = importlib.util.spec_from_file_location( - module_name, admitted_path, submodule_search_locations=search_locations + spec = importlib.util.spec_from_loader( + module_name, loader=None, origin=str(admitted_path), + is_package=search_locations is not None, ) - if spec is None or spec.loader is None: - raise ImportError("binding module loader is unavailable") + if spec is None: + raise ImportError("binding module specification is unavailable") + if search_locations is not None: + spec.submodule_search_locations = search_locations module = importlib.util.module_from_spec(spec) + module.__file__ = str(admitted_path) prior_module = sys.modules.get(module_name) sys.modules[module_name] = module def execute_import() -> None: - spec.loader.exec_module(module) + code = compile(admitted_bytes, str(admitted_path), "exec") + exec(code, module.__dict__) _run_with_timeout(execute_import) factory = getattr(module, object_name) @@ -1018,6 +1024,9 @@ def execute_import() -> None: raise ConformanceInputError("binding factory has no admitted source") from error if loaded_path != admitted_path: raise ConformanceInputError("binding factory source differs from admitted module") + post_commit, post_tree = _runtime_identity(root) + if post_commit != commit or post_tree != tree: + raise ConformanceInputError("binding repository identity changed during import") return factory, source diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 597bae8..9e85713 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -406,6 +406,35 @@ def test_binding_import_has_a_hard_timeout(self) -> None: with self.assertRaisesRegex(ConformanceInputError, "import timed out"): load_binding_factory("slow:factory", root) + def test_binding_executes_the_admitted_bytes_not_a_reopened_path(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: + root = Path(directory) + source = root / "stable.py" + source.write_text( + 'def factory():\n return "ADMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "stable.py"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + original_read = Path.read_bytes + + def substitute_after_read(path: Path) -> bytes: + payload = original_read(path) + if path.resolve() == source.resolve(): + source.write_text( + 'def factory():\n return "SUBSTITUTED"\n', encoding="utf-8" + ) + return payload + + with patch("pathlib.Path.read_bytes", substitute_after_read): + with self.assertRaisesRegex(ConformanceInputError, "dirty"): + load_binding_factory("stable:factory", root) + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From 08e3f5909030fd9713aadb729a8c89599cd06a14 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:53:22 +0200 Subject: [PATCH 85/88] fix: isolate binding package imports --- prototype/conformance.py | 94 +++++++++++++++++++++++++++++---------- spec/README.md | 2 + tests/test_conformance.py | 39 ++++++++++++++++ 3 files changed, 112 insertions(+), 23 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index da3a596..5ea00b9 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -4,6 +4,8 @@ import hashlib import importlib +import importlib.abc +import importlib.util import inspect import json import re @@ -981,26 +983,62 @@ def load_binding_factory( "sha256": hashlib.sha256(admitted_bytes).hexdigest(), } - try: - search_locations = [str(admitted_path.parent)] if admitted_path.name == "__init__.py" else None - spec = importlib.util.spec_from_loader( - module_name, loader=None, origin=str(admitted_path), - is_package=search_locations is not None, - ) - if spec is None: - raise ImportError("binding module specification is unavailable") - if search_locations is not None: - spec.submodule_search_locations = search_locations - module = importlib.util.module_from_spec(spec) - module.__file__ = str(admitted_path) - prior_module = sys.modules.get(module_name) - sys.modules[module_name] = module - - def execute_import() -> None: - code = compile(admitted_bytes, str(admitted_path), "exec") + executed_sources: dict[str, tuple[str, bytes]] = {} + + class AdmittedLoader(importlib.abc.Loader): + def __init__(self, fullname: str, path: Path, payload: bytes) -> None: + self.fullname = fullname + self.path = path + self.payload = payload + + def create_module(self, spec: object) -> None: + return None + + def exec_module(self, module: object) -> None: + relative_path = self.path.relative_to(root).as_posix() + executed_sources[self.fullname] = (relative_path, self.payload) + code = compile(self.payload, str(self.path), "exec") exec(code, module.__dict__) - _run_with_timeout(execute_import) + class AdmittedFinder(importlib.abc.MetaPathFinder): + def find_spec( + self, fullname: str, path: object = None, target: object = None + ) -> object: + if fullname != module_name and not fullname.startswith(f"{module_name}."): + return None + name_parts = fullname.split(".") + source_path = root.joinpath(*name_parts).with_suffix(".py") + init_path = root.joinpath(*name_parts, "__init__.py") + matches = [item.resolve() for item in (source_path, init_path) if item.is_file()] + if len(matches) != 1: + raise ImportError(f"binding dependency {fullname!r} is ambiguous or missing") + selected = matches[0] + selected_relative = selected.relative_to(root).as_posix() + if selected_relative not in tracked: + raise ImportError(f"binding dependency {fullname!r} is not tracked") + payload = admitted_bytes if fullname == module_name else selected.read_bytes() + loader = AdmittedLoader(fullname, selected, payload) + is_package = selected.name == "__init__.py" + return importlib.util.spec_from_file_location( + fullname, + selected, + loader=loader, + submodule_search_locations=[str(selected.parent)] if is_package else None, + ) + + namespace_prefix = f"{module_name}." + saved_modules = { + name: module + for name, module in tuple(sys.modules.items()) + if name == module_name or name.startswith(namespace_prefix) + } + for name in saved_modules: + sys.modules.pop(name, None) + finder = AdmittedFinder() + sys.meta_path.insert(0, finder) + + try: + module = _run_with_timeout(lambda: importlib.import_module(module_name)) factory = getattr(module, object_name) except _BindingTimeout as error: raise ConformanceInputError("binding import timed out") from error @@ -1011,11 +1049,12 @@ def execute_import() -> None: f"binding import failed: {type(error).__name__}" ) from error finally: - if "prior_module" in locals(): - if prior_module is None: - sys.modules.pop(module_name, None) - else: - sys.modules[module_name] = prior_module + if finder in sys.meta_path: + sys.meta_path.remove(finder) + for name in tuple(sys.modules): + if name == module_name or name.startswith(namespace_prefix): + sys.modules.pop(name, None) + sys.modules.update(saved_modules) if not callable(factory): raise ConformanceInputError("binding factory must be callable") try: @@ -1027,6 +1066,15 @@ def execute_import() -> None: post_commit, post_tree = _runtime_identity(root) if post_commit != commit or post_tree != tree: raise ConformanceInputError("binding repository identity changed during import") + dependency_digest = hashlib.sha256() + for fullname, (dependency_path, payload) in sorted(executed_sources.items()): + dependency_digest.update(fullname.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(dependency_path.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(hashlib.sha256(payload).digest()) + dependency_digest.update(b"\0") + source["dependency_manifest_sha256"] = dependency_digest.hexdigest() return factory, source diff --git a/spec/README.md b/spec/README.md index b8afb96..83c013a 100644 --- a/spec/README.md +++ b/spec/README.md @@ -45,6 +45,8 @@ For an independently implemented binding in another checkout, add `--binding module:factory --binding-root /path/to/its/clean/git/checkout`. Reports bind the LLM Errata runtime commit/tree, exact corpus SHA-256, and the binding repository commit/tree plus tracked source path and SHA-256 separately. +Binding-package modules are loaded from admitted tracked bytes rather than any +preloaded module cache, and the report binds their dependency-manifest digest. Exit `0` means the supplied binding and validator controls passed this internal corpus. Exit `1` means a behavioral or mutation control failed. Exit diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 9e85713..b302ba1 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -11,6 +11,7 @@ import sys import tempfile import time +import types import unittest from pathlib import Path from unittest.mock import patch @@ -435,6 +436,44 @@ def substitute_after_read(path: Path) -> bytes: with self.assertRaisesRegex(ConformanceInputError, "dirty"): load_binding_factory("stable:factory", root) + def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text( + "from .helper import VALUE\n" + "def factory():\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached = types.ModuleType("bindingpkg.helper") + cached.VALUE = "CACHED-UNBOUND" + prior = sys.modules.get("bindingpkg.helper") + sys.modules["bindingpkg.helper"] = cached + try: + factory, identity = load_binding_factory("bindingpkg:factory", root) + finally: + if prior is None: + sys.modules.pop("bindingpkg.helper", None) + else: + sys.modules["bindingpkg.helper"] = prior + self.assertEqual(factory(), "COMMITTED") + self.assertRegex( + identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" + ) + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From c7fa670e77b6b42314e0f25f5d0921041ac9eaa0 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 00:56:45 +0200 Subject: [PATCH 86/88] fix: isolate dotted binding namespaces --- prototype/conformance.py | 12 +++++---- tests/test_conformance.py | 51 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/prototype/conformance.py b/prototype/conformance.py index 5ea00b9..5dc716d 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -1000,11 +1000,13 @@ def exec_module(self, module: object) -> None: code = compile(self.payload, str(self.path), "exec") exec(code, module.__dict__) + top_package = parts[0] + class AdmittedFinder(importlib.abc.MetaPathFinder): def find_spec( self, fullname: str, path: object = None, target: object = None ) -> object: - if fullname != module_name and not fullname.startswith(f"{module_name}."): + if fullname != top_package and not fullname.startswith(f"{top_package}."): return None name_parts = fullname.split(".") source_path = root.joinpath(*name_parts).with_suffix(".py") @@ -1016,7 +1018,7 @@ def find_spec( selected_relative = selected.relative_to(root).as_posix() if selected_relative not in tracked: raise ImportError(f"binding dependency {fullname!r} is not tracked") - payload = admitted_bytes if fullname == module_name else selected.read_bytes() + payload = admitted_bytes if selected == admitted_path else selected.read_bytes() loader = AdmittedLoader(fullname, selected, payload) is_package = selected.name == "__init__.py" return importlib.util.spec_from_file_location( @@ -1026,11 +1028,11 @@ def find_spec( submodule_search_locations=[str(selected.parent)] if is_package else None, ) - namespace_prefix = f"{module_name}." + namespace_prefix = f"{top_package}." saved_modules = { name: module for name, module in tuple(sys.modules.items()) - if name == module_name or name.startswith(namespace_prefix) + if name == top_package or name.startswith(namespace_prefix) } for name in saved_modules: sys.modules.pop(name, None) @@ -1052,7 +1054,7 @@ def find_spec( if finder in sys.meta_path: sys.meta_path.remove(finder) for name in tuple(sys.modules): - if name == module_name or name.startswith(namespace_prefix): + if name == top_package or name.startswith(namespace_prefix): sys.modules.pop(name, None) sys.modules.update(saved_modules) if not callable(factory): diff --git a/tests/test_conformance.py b/tests/test_conformance.py index b302ba1..5f86237 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -474,6 +474,57 @@ def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> No identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" ) + def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "subbinding.py").write_text( + "from .helper import VALUE\n" + "def factory():\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached_parent = types.ModuleType("bindingpkg") + cached_parent.__path__ = [str(package)] + cached_helper = types.ModuleType("bindingpkg.helper") + cached_helper.VALUE = "CACHED-UNBOUND" + prior_parent = sys.modules.get("bindingpkg") + prior_helper = sys.modules.get("bindingpkg.helper") + sys.modules["bindingpkg"] = cached_parent + sys.modules["bindingpkg.helper"] = cached_helper + try: + factory, identity = load_binding_factory( + "bindingpkg.subbinding:factory", root + ) + self.assertIs(sys.modules["bindingpkg"], cached_parent) + self.assertIs(sys.modules["bindingpkg.helper"], cached_helper) + finally: + if prior_parent is None: + sys.modules.pop("bindingpkg", None) + else: + sys.modules["bindingpkg"] = prior_parent + if prior_helper is None: + sys.modules.pop("bindingpkg.helper", None) + else: + sys.modules["bindingpkg.helper"] = prior_helper + self.assertEqual(factory(), "COMMITTED") + self.assertRegex( + identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" + ) + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From d7f5a451613c7f8bb40eba9f3e63e16255d8b494 Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 01:03:38 +0200 Subject: [PATCH 87/88] fix: isolate binding imports for full run --- prototype/cli.py | 24 ++++++++----- prototype/conformance.py | 71 +++++++++++++++++++++------------------ tests/test_conformance.py | 70 ++++++++++++++++++++++++++++++++------ 3 files changed, 113 insertions(+), 52 deletions(-) diff --git a/prototype/cli.py b/prototype/cli.py index ded61e4..ff0a596 100644 --- a/prototype/cli.py +++ b/prototype/cli.py @@ -67,14 +67,22 @@ def cmd_adapter_conformance(ws: Workspace, args: argparse.Namespace) -> int: binding_root = args.binding_root else: binding_root = args.binding_root or args.source_root - factory, source = load_binding_factory(args.binding, binding_root) - report = validate_adapter_conformance( - args.corpus, - args.source_root, - factory, - binding_root=binding_root, - binding_source=source, - ) + with load_binding_factory(args.binding, binding_root) as (factory, source): + report = validate_adapter_conformance( + args.corpus, + args.source_root, + factory, + binding_root=binding_root, + binding_source=source, + ) + if args.binding is None: + report = validate_adapter_conformance( + args.corpus, + args.source_root, + factory, + binding_root=binding_root, + binding_source=source, + ) except ConformanceInputError as error: print(f"invalid conformance evidence: {error}", file=sys.stderr) return EXIT_INCONCLUSIVE diff --git a/prototype/conformance.py b/prototype/conformance.py index 5dc716d..4d03190 100644 --- a/prototype/conformance.py +++ b/prototype/conformance.py @@ -13,6 +13,7 @@ import subprocess import sys import threading +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any, Callable @@ -944,11 +945,12 @@ def _binding_source( } +@contextmanager def load_binding_factory( value: str, binding_root: Path, -) -> tuple[Callable[[], ReferenceConformanceBinding], dict[str, str]]: - """Admit tracked module bytes before bounded import executes them.""" +) -> Any: + """Keep an admitted package namespace isolated for the complete run.""" try: module_name, object_name = value.split(":", 1) @@ -1040,16 +1042,40 @@ def find_spec( sys.meta_path.insert(0, finder) try: - module = _run_with_timeout(lambda: importlib.import_module(module_name)) - factory = getattr(module, object_name) - except _BindingTimeout as error: - raise ConformanceInputError("binding import timed out") from error - except BaseException as error: - if isinstance(error, KeyboardInterrupt): - raise - raise ConformanceInputError( - f"binding import failed: {type(error).__name__}" - ) from error + try: + module = _run_with_timeout(lambda: importlib.import_module(module_name)) + factory = getattr(module, object_name) + except _BindingTimeout as error: + raise ConformanceInputError("binding import timed out") from error + except BaseException as error: + if isinstance(error, KeyboardInterrupt): + raise + raise ConformanceInputError( + f"binding import failed: {type(error).__name__}" + ) from error + if not callable(factory): + raise ConformanceInputError("binding factory must be callable") + try: + loaded_path = Path(inspect.getsourcefile(factory) or "").resolve() + except (OSError, TypeError, ValueError) as error: + raise ConformanceInputError("binding factory has no admitted source") from error + if loaded_path != admitted_path: + raise ConformanceInputError("binding factory source differs from admitted module") + + yield factory, source + + post_commit, post_tree = _runtime_identity(root) + if post_commit != commit or post_tree != tree: + raise ConformanceInputError("binding repository identity changed during execution") + dependency_digest = hashlib.sha256() + for fullname, (dependency_path, payload) in sorted(executed_sources.items()): + dependency_digest.update(fullname.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(dependency_path.encode("utf-8")) + dependency_digest.update(b"\0") + dependency_digest.update(hashlib.sha256(payload).digest()) + dependency_digest.update(b"\0") + source["dependency_manifest_sha256"] = dependency_digest.hexdigest() finally: if finder in sys.meta_path: sys.meta_path.remove(finder) @@ -1057,27 +1083,6 @@ def find_spec( if name == top_package or name.startswith(namespace_prefix): sys.modules.pop(name, None) sys.modules.update(saved_modules) - if not callable(factory): - raise ConformanceInputError("binding factory must be callable") - try: - loaded_path = Path(inspect.getsourcefile(factory) or "").resolve() - except (OSError, TypeError, ValueError) as error: - raise ConformanceInputError("binding factory has no admitted source") from error - if loaded_path != admitted_path: - raise ConformanceInputError("binding factory source differs from admitted module") - post_commit, post_tree = _runtime_identity(root) - if post_commit != commit or post_tree != tree: - raise ConformanceInputError("binding repository identity changed during import") - dependency_digest = hashlib.sha256() - for fullname, (dependency_path, payload) in sorted(executed_sources.items()): - dependency_digest.update(fullname.encode("utf-8")) - dependency_digest.update(b"\0") - dependency_digest.update(dependency_path.encode("utf-8")) - dependency_digest.update(b"\0") - dependency_digest.update(hashlib.sha256(payload).digest()) - dependency_digest.update(b"\0") - source["dependency_manifest_sha256"] = dependency_digest.hexdigest() - return factory, source class _BindingTimeout(Exception): diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 5f86237..704644a 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -384,7 +384,8 @@ def test_binding_import_runtime_error_is_invalid_evidence(self) -> None: subprocess.run(command, cwd=root, check=True) with patch("sys.path", [str(root), *sys.path]): with self.assertRaisesRegex(ConformanceInputError, "RuntimeError"): - load_binding_factory("broken:factory", root) + with load_binding_factory("broken:factory", root): + pass def test_binding_import_has_a_hard_timeout(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: @@ -405,7 +406,8 @@ def test_binding_import_has_a_hard_timeout(self) -> None: "prototype.conformance.BINDING_TIMEOUT_SECONDS", 0.01 ): with self.assertRaisesRegex(ConformanceInputError, "import timed out"): - load_binding_factory("slow:factory", root) + with load_binding_factory("slow:factory", root): + pass def test_binding_executes_the_admitted_bytes_not_a_reopened_path(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-binding-import-") as directory: @@ -434,7 +436,8 @@ def substitute_after_read(path: Path) -> bytes: with patch("pathlib.Path.read_bytes", substitute_after_read): with self.assertRaisesRegex(ConformanceInputError, "dirty"): - load_binding_factory("stable:factory", root) + with load_binding_factory("stable:factory", root) as (factory, _): + self.assertEqual(factory(), "ADMITTED") def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: @@ -463,16 +466,16 @@ def test_preloaded_package_dependency_cannot_inject_unbound_behavior(self) -> No prior = sys.modules.get("bindingpkg.helper") sys.modules["bindingpkg.helper"] = cached try: - factory, identity = load_binding_factory("bindingpkg:factory", root) + with load_binding_factory("bindingpkg:factory", root) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertRegex( + identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" + ) finally: if prior is None: sys.modules.pop("bindingpkg.helper", None) else: sys.modules["bindingpkg.helper"] = prior - self.assertEqual(factory(), "COMMITTED") - self.assertRegex( - identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" - ) def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: @@ -506,9 +509,12 @@ def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: sys.modules["bindingpkg"] = cached_parent sys.modules["bindingpkg.helper"] = cached_helper try: - factory, identity = load_binding_factory( + with load_binding_factory( "bindingpkg.subbinding:factory", root - ) + ) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertIsNot(sys.modules["bindingpkg"], cached_parent) + self.assertIsNot(sys.modules["bindingpkg.helper"], cached_helper) self.assertIs(sys.modules["bindingpkg"], cached_parent) self.assertIs(sys.modules["bindingpkg.helper"], cached_helper) finally: @@ -520,11 +526,53 @@ def test_dotted_binding_isolates_parent_and_sibling_module_cache(self) -> None: sys.modules.pop("bindingpkg.helper", None) else: sys.modules["bindingpkg.helper"] = prior_helper - self.assertEqual(factory(), "COMMITTED") self.assertRegex( identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$" ) + def test_lazy_dependency_import_remains_isolated_for_factory_lifetime(self) -> None: + with tempfile.TemporaryDirectory(prefix="errata-binding-package-") as directory: + root = Path(directory) + package = root / "bindingpkg" + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "subbinding.py").write_text( + "def factory():\n" + " from .helper import VALUE\n" + " return VALUE\n", + encoding="utf-8", + ) + (package / "helper.py").write_text( + 'VALUE = "COMMITTED"\n', encoding="utf-8" + ) + for command in ( + ("git", "init", "-q"), + ("git", "config", "user.email", "tests@example.invalid"), + ("git", "config", "user.name", "Conformance Tests"), + ("git", "add", "bindingpkg"), + ("git", "commit", "-q", "-m", "binding"), + ): + subprocess.run(command, cwd=root, check=True) + cached_parent = types.ModuleType("bindingpkg") + cached_parent.__path__ = [str(package)] + cached_helper = types.ModuleType("bindingpkg.helper") + cached_helper.VALUE = "CACHED-UNBOUND" + saved = {name: sys.modules.get(name) for name in ("bindingpkg", "bindingpkg.helper")} + sys.modules["bindingpkg"] = cached_parent + sys.modules["bindingpkg.helper"] = cached_helper + try: + with load_binding_factory( + "bindingpkg.subbinding:factory", root + ) as (factory, identity): + self.assertEqual(factory(), "COMMITTED") + self.assertRegex(identity["dependency_manifest_sha256"], r"^[0-9a-f]{64}$") + finally: + for name, module in saved.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + def test_dirty_runtime_tree_is_refused_before_binding_execution(self) -> None: with tempfile.TemporaryDirectory(prefix="errata-dirty-source-") as directory: source_root = Path(directory) From 9eba33fc341ea596f972292da0e4097eb2ea632b Mon Sep 17 00:00:00 2001 From: Thomas Willner Date: Fri, 14 Aug 2026 01:16:39 +0200 Subject: [PATCH 88/88] test: disable background gc in disposable repos --- tests/test_readiness.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 20177e5..88ab599 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -99,6 +99,7 @@ def test_g6_complete_measured_report_is_commit_and_deployment_bound(self) -> Non ("git", "init"), ("git", "config", "user.email", "tests@example.invalid"), ("git", "config", "user.name", "Readiness tests"), + ("git", "config", "gc.auto", "0"), ("git", "add", "."), ("git", "commit", "-m", "operational baseline"), ): @@ -119,6 +120,7 @@ def test_g6_failed_comparator_is_valid_but_not_qualifying(self) -> None: ("git", "init"), ("git", "config", "user.email", "tests@example.invalid"), ("git", "config", "user.name", "Readiness tests"), + ("git", "config", "gc.auto", "0"), ("git", "add", "."), ("git", "commit", "-m", "operational baseline"), ): @@ -140,6 +142,7 @@ def test_g6_report_rejects_missing_or_ambiguous_measurement_evidence(self) -> No ("git", "init"), ("git", "config", "user.email", "tests@example.invalid"), ("git", "config", "user.name", "Readiness tests"), + ("git", "config", "gc.auto", "0"), ("git", "add", "."), ("git", "commit", "-m", "operational baseline"), ): @@ -247,6 +250,7 @@ def test_g2_review_binds_current_committed_surface(self) -> None: ("git", "init"), ("git", "config", "user.email", "tests@example.invalid"), ("git", "config", "user.name", "Readiness tests"), + ("git", "config", "gc.auto", "0"), ("git", "add", "."), ("git", "commit", "-m", "surface baseline"), ): @@ -275,6 +279,7 @@ def test_g2_review_rejects_nonexistent_commit(self) -> None: ("git", "init"), ("git", "config", "user.email", "tests@example.invalid"), ("git", "config", "user.name", "Readiness tests"), + ("git", "config", "gc.auto", "0"), ("git", "add", "."), ("git", "commit", "-m", "surface baseline"), ):