From 7713df53a618453cb5abae072959a62fa1c1c844 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:00:59 -0700 Subject: [PATCH 1/3] test(runtime): map API-404 clauses to each control-plane profile's evidence Add a clause-by-profile verification map to the conformance page, cover run binding on every profile, and pin the map to its clauses, the profiles' declared guarantees and its cited files. The conformance witness now records evaluator starts, so the run-binding cases observe backend effects as well as records, audit and revisions. --- docs/requirements/API-404/requirement.md | 4 +- .../runtime-control-plane/conformance.md | 66 +++++++ .../control_plane_conformance_fixtures.py | 20 ++- ..._issue_1435_profile_clause_verification.py | 170 ++++++++++++++++++ 4 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 implementations/python/tests/test_issue_1435_profile_clause_verification.py diff --git a/docs/requirements/API-404/requirement.md b/docs/requirements/API-404/requirement.md index 4ae390a9e..552c9cfe6 100644 --- a/docs/requirements/API-404/requirement.md +++ b/docs/requirements/API-404/requirement.md @@ -213,7 +213,7 @@ identifies those implementation gaps and the retained canonical requirements. - DOCUMENTS → GITHUB_ISSUE `1188` (CP-8: Served profile alignment) - DOCUMENTS → GITHUB_ISSUE `1187` (CP-9: Crash and profile conformance suite) - DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1187-control-plane-conformance-preflight.md` (CP-9 profile, fault-evidence, and security boundaries) -- DOCUMENTS → DOCUMENTATION `docs/research/runtime-control-plane/conformance.md` (Unified suite invocation, finite evidence map, and explicit profile nonclaims) +- DOCUMENTS → DOCUMENTATION `docs/research/runtime-control-plane/conformance.md` (Unified suite invocation, finite evidence map, API-404 clause-to-profile verification map, and explicit profile nonclaims) - IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/backend_result_diagnostics.py` (Value-free backend failure diagnostics without provider type names) - IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/observation_execution.py` (Observation-adapter failure diagnostics without provider exception names or text) - TESTS → TEST `implementations/python/tests/test_issue_1212_observation_demand.py` (Observation failures preserve terminal state and redact exception details from operation carriers) @@ -238,6 +238,8 @@ identifies those implementation gaps and the retained canonical requirements. - DOCUMENTS → GITHUB_ISSUE `1185` (CP-11: API-404 requirement update) - DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1185-api-404-profile-alignment-preflight.md` (CP-11 profile and evidence boundaries) - TESTS → TEST `implementations/python/tests/test_issue_1185_api_404_profile_alignment.py` (Clause matrix, profile catalog, nonclaims, public documentation, and local-evidence drift checks) +- DOCUMENTS → GITHUB_ISSUE `1435` (API-404 clause-to-profile evidence map and run binding on every profile) +- TESTS → TEST `implementations/python/tests/test_issue_1435_profile_clause_verification.py` (Out-of-run requests refused without record, evaluator effect, audit, or revision change on P0, P1, and P2; P0 run rebinding refused; clause-map profiles checked against profile declarations; citation and link drift checks) - DOCUMENTS → GITHUB_ISSUE `1186` (CP-12: Recovery runbook and operator tooling) - DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1186-control-plane-recovery-operations-preflight.md` (CP-12 recovery, maintenance, health, and disclosure boundaries) - DOCUMENTS → DOCUMENTATION `docs/explain/sdl/control-plane-operations.md` (Operator recovery, health, shutdown, backup, restore, and upgrade runbook) diff --git a/docs/research/runtime-control-plane/conformance.md b/docs/research/runtime-control-plane/conformance.md index 99f77e853..910905de6 100644 --- a/docs/research/runtime-control-plane/conformance.md +++ b/docs/research/runtime-control-plane/conformance.md @@ -69,3 +69,69 @@ external-effect, tamper-evidence, or universal recovery-proof claim. Payload digests detect corruption; a writer able to replace both content and digest is not thereby authenticated. ASGI tests do not verify deployment TLS, trusted-proxy header stripping, external backend recovery, or a matching backend backup. + +## API-404 clause verification + +[API-404](../../requirements/API-404/requirement.md) states the clauses and +the profiles each clause binds. `profile_declaration()` holds each profile's +guarantee and nonclaim identifiers. The table maps each clause to the landed +code, conformance cases, and operator guidance behind it. Code paths are +relative to `implementations/python/packages/raes_runtime/`, and test paths to +`implementations/python/tests/`. + +| Clause | Profiles | Implementation | Conformance cases | Operator guidance | +| --- | --- | --- | --- | --- | +| `API-404-C1` | P0, P1, P2 | `control_plane.py`, `control_plane_operation_context.py`, `control_plane_admission.py`, `control_plane_mutation.py`, `control_plane_store.py`, `control_plane_store_memory.py` | `test_issue_1187_control_plane_profiles.py`, `test_issue_1187_control_plane_lifecycle_properties.py`, `test_issue_1184_atomic_idempotency_claims.py`, `test_issue_1180_snapshot_revision_cas.py`, `test_issue_1181_unified_control_plane_mutations.py`, `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1435_profile_clause_verification.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md) | +| `API-404-C2` | P1, P2 | `control_plane_store_local.py`, `control_plane_store_local_scope.py`, `control_plane_store_lease.py`, `control_plane_durability.py`, `control_plane_recovery.py` | `test_issue_1187_control_plane_process_loss.py`, `test_issue_1187_control_plane_durable_carriers.py`, `test_issue_1179_startup_reconciliation.py`, `test_issue_1183_store_ownership_leases.py`, `test_issue_1092_control_plane_crash_consistency.py`, `test_issue_1187_control_plane_profiles.py` | [Control-plane operations](../../explain/sdl/control-plane-operations.md) | +| `API-404-C3` | P2 | `control_plane_api/__init__.py`, `control_plane_api/_auth.py`, `control_plane_api/_offload.py`, `control_plane_api_guards.py`, `control_plane_security.py` | `test_issue_1187_control_plane_security_conformance.py`, `test_issue_1359_runtime_api_trust_boundary.py`, `test_issue_1093_request_rejection_offload.py`, `test_runtime_control_plane_api.py`, `test_issue_1187_control_plane_profiles.py` | [Serve the runtime control plane](../../public/guides/control-plane.md) | +| `API-404-C4` | P0, P1, P2; P3 is unavailable | `control_plane_profiles.py` | `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1185_api_404_profile_alignment.py`, `test_issue_1187_control_plane_process_loss.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md) and the limits above | + +Each property #8 names is checked on every profile that guarantees it: + +- **Actor binding.** Retry, readback and disclosure stay with the submitting + actor on P0, P1 and P2 + (`test_issue_1187_control_plane_profiles.py::test_profile_scoped_retry_status_audit_and_cache_coherence`). +- **Target and run binding.** A selected P0 store refuses another target + (`test_issue_1189_control_plane_profile_declarations.py::test_selected_p0_uses_canonical_declaration_and_binds_one_scope`) + and another run + (`test_issue_1435_profile_clause_verification.py::test_selected_p0_store_cannot_be_rebound_to_another_run`). + A P1 or P2 store refuses both + (`test_issue_1187_control_plane_profiles.py::test_profile_lease_scope_and_restore_keep_receipts_and_claims`). + On every profile, a request for another run leaves no record, effect, audit + or revision change. The same request for the admitted run adds one record, + one terminal audit and one evaluator start in the backend-effect witness, and + moves the snapshot revision from 0 to 1 + (`test_issue_1435_profile_clause_verification.py::test_request_for_another_run_changes_nothing`). + The refusal is not an admission denial, which creates audit only under the + [CP-9 preflight](../../decisions/issue-1187-control-plane-conformance-preflight.md). + The [CP-5 preflight](../../decisions/issue-1183-store-ownership-lease-preflight.md) + requires a request for another run to fail before claim or disclosure, and + `operation_admission_context()` refuses it before building the operation + context that an admission-denial audit records. +- **Atomic state and audit.** Every operation kind commits its terminal state + with exactly one actor-bound audit on P0 and P1 + (`test_issue_1187_control_plane_lifecycle_properties.py::test_each_operation_kind_commits_one_immutable_actor_bound_terminal_cut`), + and so does an HTTP submission on P2 + (`test_issue_1187_control_plane_profiles.py::test_profile_scoped_retry_status_audit_and_cache_coherence`). +- **Retained idempotency.** P1 and P2 keep receipts and claims across restore + (`test_issue_1187_control_plane_profiles.py::test_profile_lease_scope_and_restore_keep_receipts_and_claims`). + P0 loses them with its process + (`test_issue_1187_control_plane_process_loss.py::test_p0_process_loss_loses_run_and_claim_even_when_external_effect_survives`). +- **Ownership.** A P0 store has one live owner + (`test_issue_1189_control_plane_profile_declarations.py::test_selected_p0_store_has_one_active_owner_and_releases_it_on_close`). + P1 and P2 admit one lease owner and refuse a second + (`test_issue_1187_control_plane_profiles.py::test_profile_lease_scope_and_restore_keep_receipts_and_claims`). +- **Revision checks.** A stale snapshot write fails on every profile + (`test_issue_1187_control_plane_profiles.py::test_profile_scoped_retry_status_audit_and_cache_coherence`). +- **Startup classification.** P1 and P2 classify interrupted work without + replay + (`test_issue_1187_control_plane_process_loss.py::test_process_loss_preserves_atomic_cut_and_classifies_without_replay`). + It does not apply to P0, which keeps no state across process loss. + +P2 serves a selected P1 core and adds no store of its own, so its C1 and C2 +evidence is the P1 store evidence plus the HTTP cases. +`test_issue_1435_profile_clause_verification.py` derives each clause's profiles +from `profile_declaration()`: the available profiles whose guarantees include +the clause's identifiers, or every available profile for C4. It fails if that +set, this table and an independently written expectation disagree, or if the +section cites a file, test or link that does not exist. diff --git a/implementations/python/tests/control_plane_conformance_fixtures.py b/implementations/python/tests/control_plane_conformance_fixtures.py index 48f06a814..1f3020b0d 100644 --- a/implementations/python/tests/control_plane_conformance_fixtures.py +++ b/implementations/python/tests/control_plane_conformance_fixtures.py @@ -15,7 +15,7 @@ RecoveryObservationRequest, RecoveryObservationResult, ) -from raes_backend_stubs.stubs import create_stub_target +from raes_backend_stubs.stubs import StubEvaluator, create_stub_target from raes_contracts.plan_projection import provisioning_plan_model from raes_contracts.planning import ProvisioningPlan from raes_contracts.runtime_state import OperationKind, OperationState @@ -81,6 +81,18 @@ def apply(self, plan, snapshot): return result +class WitnessEvaluator(StubEvaluator): + # An evaluator start is a backend effect too, so it joins the same witness. + def __init__(self, witness: Path) -> None: + super().__init__() + self.witness = witness + + def start(self, plan, snapshot): + assert plan.operation_id + record_effect(self.witness, "evaluate", plan.operation_id) + return super().start(plan, snapshot) + + class WitnessObserver: def __init__(self, witness: Path) -> None: self.witness = witness @@ -109,7 +121,11 @@ def witness_target(witness: Path, *, observe: bool = False, boundary=None): # Recovery intentionally refuses to invent a provisioning plan to authorize # new realization/resource state from an observer's assertion alone. target = create_stub_target(with_realization_envelope=False) - target = replace(target, provisioner=WitnessProvisioner(target.provisioner, witness, boundary)) + target = replace( + target, + provisioner=WitnessProvisioner(target.provisioner, witness, boundary), + evaluator=WitnessEvaluator(witness), + ) if observe: capability = RecoveryObservationCapabilities( name="conformance-witness", supported_operation_kinds=frozenset({OperationKind.PROVISIONING}) diff --git a/implementations/python/tests/test_issue_1435_profile_clause_verification.py b/implementations/python/tests/test_issue_1435_profile_clause_verification.py new file mode 100644 index 000000000..fc00d6a70 --- /dev/null +++ b/implementations/python/tests/test_issue_1435_profile_clause_verification.py @@ -0,0 +1,170 @@ +"""API-404 clause verification against each control-plane profile (#1435, for #8). + +``docs/research/runtime-control-plane/conformance.md`` maps every API-404 +clause to the landed code, conformance cases and operator guidance of the +profiles it binds. These cases cover the run binding that no profile case +exercised, and keep that map tied to its clauses, the profiles' declared +guarantees and its cited files. +""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +import pytest +from control_plane_conformance_fixtures import ( + KEY, + PROFILES, + RUN_SCOPE, + ProfileHarness, + identities, + profile_harness, + terminal_audits, + witness_events, +) +from raes_backend_stubs.stubs import create_stub_target +from raes_contracts.plan_projection import evaluation_plan_model +from raes_contracts.planning import EvaluationPlan +from raes_runtime.control_plane import RuntimeControlPlane +from raes_runtime.control_plane_profiles import ControlPlaneProfile, profile_declaration +from raes_runtime.control_plane_store_memory import InMemoryControlPlaneStore +from test_issue_1185_api_404_profile_alignment import COMMON_GUARANTEES, DURABLE_GUARANTEES, TRANSPORT_GUARANTEES + +pytestmark = pytest.mark.control_plane_conformance + +ROOT = Path(__file__).resolve().parents[3] +CONFORMANCE_PAGE = ROOT / "docs/research/runtime-control-plane/conformance.md" +CODE_ROOT = ROOT / "implementations/python/packages/raes_runtime" +TEST_ROOT = Path(__file__).resolve().parent +ADMITTED_RUN = RUN_SCOPE.removeprefix("run:") +ACCEPTED = "accepted" +# How each composition refuses a request whose run differs from its admitted run. +RUN_REFUSALS = { + "P0": "operation run scope does not match the admitted control-plane store scope", + "P1": "operation run scope does not match the admitted control-plane store scope", + "P2": '409 {"detail":"operation conflict"}', +} +# Written independently of the requirement document: the profiles each clause binds. +CLAUSE_PROFILES = { + "API-404-C1": {"P0", "P1", "P2"}, + "API-404-C2": {"P1", "P2"}, + "API-404-C3": {"P2"}, + "API-404-C4": {"P0", "P1", "P2"}, +} +# The guarantee identifiers API-404 names for each clause, as the #1185 module +# pins them. C4 names none, so it binds every available profile. +CLAUSE_GUARANTEES = { + "API-404-C1": COMMON_GUARANTEES, + "API-404-C2": DURABLE_GUARANTEES, + "API-404-C3": TRANSPORT_GUARANTEES, + "API-404-C4": set(), +} +_CITED_FILE = re.compile(r"`([\w/]+\.py)(?:::(\w+))?`") +_LINK_TARGET = re.compile(r"\]\(([^)#\s]+)\)") + + +def _evaluate(harness: ProfileHarness, run_id: str) -> str: + """Submit one evaluation plan through the composition's own entry point.""" + + plan = EvaluationPlan(run_id=run_id) + if harness.client is not None: + response = harness.client.post( + "/operations/evaluation", + json=evaluation_plan_model(plan).model_dump(mode="json"), + headers=harness.headers(), + ) + accepted = response.status_code == 200 and response.json()["accepted"] + return ACCEPTED if accepted else f"{response.status_code} {response.text}" + try: + receipt = harness.plane.submit_evaluation(plan, idempotency_key=KEY, identity=identities()["alice"]) + except ValueError as refusal: + return str(refusal) + return ACCEPTED if receipt.accepted else repr(receipt.diagnostics) + + +@pytest.mark.parametrize("profile", PROFILES) +def test_request_for_another_run_changes_nothing(profile: str, tmp_path: Path) -> None: + witness = tmp_path / "effects.jsonl" + with profile_harness(profile, tmp_path) as harness: + assert _evaluate(harness, "another-run") == RUN_REFUSALS[profile] + + assert harness.store.load_records() == {} + assert harness.store.read_audit() == [] + assert witness_events(witness) == [] + assert harness.snapshot_cut() == ({}, 0) + + # The same plan for the admitted run is accepted, and the same reads see + # its record, terminal audit, evaluator start and revision. + assert _evaluate(harness, ADMITTED_RUN) == ACCEPTED + (operation_id,) = harness.store.load_records() + assert len(terminal_audits(harness.store, operation_id)) == 1 + assert witness_events(witness) == [{"event": "evaluate", "operation_id": operation_id}] + assert harness.snapshot_cut()[1] == 1 + + +def test_selected_p0_store_cannot_be_rebound_to_another_run() -> None: + store = InMemoryControlPlaneStore() + RuntimeControlPlane(create_stub_target(), store=store, run_scope=RUN_SCOPE, profile=ControlPlaneProfile.P0).close() + target = create_stub_target() + + with pytest.raises(ValueError, match="store scope does not match runtime admission"): + RuntimeControlPlane(target, store=store, run_scope="run:another", profile=ControlPlaneProfile.P0) + + # The refusal leaves the store bound to its first scope, which reopens. + RuntimeControlPlane(create_stub_target(), store=store, run_scope=RUN_SCOPE, profile=ControlPlaneProfile.P0).close() + + +def _clause_section() -> str: + page = CONFORMANCE_PAGE.read_text(encoding="utf-8") + return page.split("\n## API-404 clause verification\n", 1)[1].split("\n## ", 1)[0] + + +def _clause_rows() -> dict[str, list[str]]: + rows = [line.strip().strip("|").split("|") for line in _clause_section().splitlines() if line.startswith("| `API")] + return {cells[0].strip().strip("`"): [cell.strip() for cell in cells] for cells in rows} + + +def _declared_profiles(guarantees: set[str]) -> set[str]: + """Available profiles whose declared guarantees include every given identifier.""" + + declarations = [profile_declaration(profile) for profile in ControlPlaneProfile] + return { + declaration.profile.value + for declaration in declarations + if declaration.available and guarantees <= {claim.identifier for claim in declaration.guarantees} + } + + +def test_clause_map_binds_each_clause_to_its_declared_profiles() -> None: + declared = {clause: _declared_profiles(guarantees) for clause, guarantees in CLAUSE_GUARANTEES.items()} + # A note after ";" in a cell, such as "P3 is unavailable", binds no profile. + table = { + clause: set(re.findall(r"\bP\d\b", cells[1].partition(";")[0])) for clause, cells in _clause_rows().items() + } + + assert declared == CLAUSE_PROFILES + assert table == CLAUSE_PROFILES + + +def _cites_existing(path: str, function: str) -> bool: + source = (TEST_ROOT if Path(path).name.startswith("test_") else CODE_ROOT) / path + if not source.is_file(): + return False + tree = ast.parse(source.read_text(encoding="utf-8")) + return not function or function in {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} + + +def test_clause_map_cites_existing_code_and_tests() -> None: + citations = sorted(set(_CITED_FILE.findall(_clause_section()))) + + assert len(citations) > len(CLAUSE_PROFILES) + assert [citation for citation in citations if not _cites_existing(*citation)] == [] + + +def test_clause_map_links_resolve() -> None: + targets = _LINK_TARGET.findall(_clause_section()) + + assert targets + assert [target for target in targets if not (CONFORMANCE_PAGE.parent / target).is_file()] == [] From 35daf6c90d8d1425b211e7ceb2b8dce32ae185c6 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:55:52 -0700 Subject: [PATCH 2/3] test(runtime): cite the revision-bearing read routes in the API-404 clause map The C3 row now names the modules behind revision-bearing reads and redacted conflicts: control_plane_api/_operation_routes.py, control_plane_api/_responses.py and control_plane_api_participant_retrieval.py. The C1 row adds control_plane_execution.py, and the C1 and C4 guidance links point to the profiles section. The drift checks now require every Implementation and Conformance entry to be a citation they read, and they resolve heading fragments in links. --- .../runtime-control-plane/conformance.md | 12 ++++---- ..._issue_1435_profile_clause_verification.py | 30 +++++++++++++++---- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/docs/research/runtime-control-plane/conformance.md b/docs/research/runtime-control-plane/conformance.md index 910905de6..43e45b065 100644 --- a/docs/research/runtime-control-plane/conformance.md +++ b/docs/research/runtime-control-plane/conformance.md @@ -81,10 +81,10 @@ relative to `implementations/python/packages/raes_runtime/`, and test paths to | Clause | Profiles | Implementation | Conformance cases | Operator guidance | | --- | --- | --- | --- | --- | -| `API-404-C1` | P0, P1, P2 | `control_plane.py`, `control_plane_operation_context.py`, `control_plane_admission.py`, `control_plane_mutation.py`, `control_plane_store.py`, `control_plane_store_memory.py` | `test_issue_1187_control_plane_profiles.py`, `test_issue_1187_control_plane_lifecycle_properties.py`, `test_issue_1184_atomic_idempotency_claims.py`, `test_issue_1180_snapshot_revision_cas.py`, `test_issue_1181_unified_control_plane_mutations.py`, `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1435_profile_clause_verification.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md) | +| `API-404-C1` | P0, P1, P2 | `control_plane.py`, `control_plane_operation_context.py`, `control_plane_admission.py`, `control_plane_mutation.py`, `control_plane_execution.py`, `control_plane_store.py`, `control_plane_store_memory.py` | `test_issue_1187_control_plane_profiles.py`, `test_issue_1187_control_plane_lifecycle_properties.py`, `test_issue_1184_atomic_idempotency_claims.py`, `test_issue_1180_snapshot_revision_cas.py`, `test_issue_1181_unified_control_plane_mutations.py`, `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1435_profile_clause_verification.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md#control-plane-operating-profiles) | | `API-404-C2` | P1, P2 | `control_plane_store_local.py`, `control_plane_store_local_scope.py`, `control_plane_store_lease.py`, `control_plane_durability.py`, `control_plane_recovery.py` | `test_issue_1187_control_plane_process_loss.py`, `test_issue_1187_control_plane_durable_carriers.py`, `test_issue_1179_startup_reconciliation.py`, `test_issue_1183_store_ownership_leases.py`, `test_issue_1092_control_plane_crash_consistency.py`, `test_issue_1187_control_plane_profiles.py` | [Control-plane operations](../../explain/sdl/control-plane-operations.md) | -| `API-404-C3` | P2 | `control_plane_api/__init__.py`, `control_plane_api/_auth.py`, `control_plane_api/_offload.py`, `control_plane_api_guards.py`, `control_plane_security.py` | `test_issue_1187_control_plane_security_conformance.py`, `test_issue_1359_runtime_api_trust_boundary.py`, `test_issue_1093_request_rejection_offload.py`, `test_runtime_control_plane_api.py`, `test_issue_1187_control_plane_profiles.py` | [Serve the runtime control plane](../../public/guides/control-plane.md) | -| `API-404-C4` | P0, P1, P2; P3 is unavailable | `control_plane_profiles.py` | `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1185_api_404_profile_alignment.py`, `test_issue_1187_control_plane_process_loss.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md) and the limits above | +| `API-404-C3` | P2 | `control_plane_api/__init__.py`, `control_plane_api/_auth.py`, `control_plane_api/_offload.py`, `control_plane_api/_operation_routes.py`, `control_plane_api/_responses.py`, `control_plane_api_guards.py`, `control_plane_api_participant_retrieval.py`, `control_plane_security.py` | `test_issue_1187_control_plane_security_conformance.py`, `test_issue_1359_runtime_api_trust_boundary.py`, `test_issue_1093_request_rejection_offload.py`, `test_runtime_control_plane_api.py`, `test_issue_1187_control_plane_profiles.py` | [Serve the runtime control plane](../../public/guides/control-plane.md) | +| `API-404-C4` | P0, P1, P2; P3 is unavailable | `control_plane_profiles.py` | `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1185_api_404_profile_alignment.py`, `test_issue_1187_control_plane_process_loss.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md#control-plane-operating-profiles) and the limits above | Each property #8 names is checked on every profile that guarantees it: @@ -133,5 +133,7 @@ evidence is the P1 store evidence plus the HTTP cases. `test_issue_1435_profile_clause_verification.py` derives each clause's profiles from `profile_declaration()`: the available profiles whose guarantees include the clause's identifiers, or every available profile for C4. It fails if that -set, this table and an independently written expectation disagree, or if the -section cites a file, test or link that does not exist. +set, this table and an independently written expectation disagree. It also +fails if an Implementation or Conformance cell holds an entry that is not a +cited file, if the section cites a file or test that does not exist, or if a +link names a missing page or heading. diff --git a/implementations/python/tests/test_issue_1435_profile_clause_verification.py b/implementations/python/tests/test_issue_1435_profile_clause_verification.py index fc00d6a70..21935bba7 100644 --- a/implementations/python/tests/test_issue_1435_profile_clause_verification.py +++ b/implementations/python/tests/test_issue_1435_profile_clause_verification.py @@ -62,7 +62,10 @@ "API-404-C4": set(), } _CITED_FILE = re.compile(r"`([\w/]+\.py)(?:::(\w+))?`") -_LINK_TARGET = re.compile(r"\]\(([^)#\s]+)\)") +# A link's page, empty for this page, and its optional heading fragment. +_LINK_TARGET = re.compile(r"\]\(([^)#\s]*)(?:#([^)\s]*))?\)") +_HEADING = re.compile(r"^#+ (.+)$", re.MULTILINE) +_NOT_IN_ANCHOR = re.compile(r"[^\w -]") def _evaluate(harness: ProfileHarness, run_id: str) -> str: @@ -157,14 +160,31 @@ def _cites_existing(path: str, function: str) -> bool: def test_clause_map_cites_existing_code_and_tests() -> None: + rows = _clause_rows() + # Every entry of each Implementation and Conformance cell must be a citation + # the pattern reads, so an emptied cell or a narrowed pattern fails here. + entries = [entry for cells in rows.values() for cell in (cells[2], cells[3]) for entry in cell.split(", ")] citations = sorted(set(_CITED_FILE.findall(_clause_section()))) - assert len(citations) > len(CLAUSE_PROFILES) + assert rows.keys() == CLAUSE_PROFILES.keys() + assert [entry for entry in entries if not _CITED_FILE.fullmatch(entry)] == [] assert [citation for citation in citations if not _cites_existing(*citation)] == [] +def _heading_anchors(page: Path) -> set[str]: + """The fragments GitHub derives from a page's headings, without duplicate suffixes.""" + + headings = _HEADING.findall(page.read_text(encoding="utf-8")) + return {_NOT_IN_ANCHOR.sub("", heading.strip().lower()).replace(" ", "-") for heading in headings} + + +def _link_resolves(target: str, fragment: str) -> bool: + page = CONFORMANCE_PAGE.parent / target if target else CONFORMANCE_PAGE + return page.is_file() and (not fragment or fragment in _heading_anchors(page)) + + def test_clause_map_links_resolve() -> None: - targets = _LINK_TARGET.findall(_clause_section()) + links = _LINK_TARGET.findall(_clause_section()) - assert targets - assert [target for target in targets if not (CONFORMANCE_PAGE.parent / target).is_file()] == [] + assert links + assert [link for link in links if not _link_resolves(*link)] == [] From 2ef934ff89f56f59445fc90699dcbfca132008a6 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Sat, 10 Oct 2026 09:04:58 -0700 Subject: [PATCH 3/3] test(runtime): narrow the out-of-run audit claim in the API-404 clause map An evaluation plan for another run that carries no operations or observation demands leaves no audit on any profile. On P2, the provisioning, orchestration and evaluation routes check planner authorization first. A new case sends an unregistered plan with one operation for another run to each composition: P2 answers 403 with the route's planner-authorization-mismatch denial audit, P0 and P1 refuse it on the run check with no audit, and no profile adds a record or effect or changes the revision. The page cites the CP-2 preflight's admission-denial definition for the run refusal, and the API-404 trace line is narrowed to match. The C1 row adds control_plane_configuration.py, control_plane_store_revision.py and control_plane_store_local_snapshot.py. The C2 row adds control_plane_store_local_records.py, control_plane_store_local_codec.py and control_plane_store_records.py. The module docstring names the run binding the new cases add. --- docs/requirements/API-404/requirement.md | 2 +- .../runtime-control-plane/conformance.md | 32 +++++++++----- ..._issue_1435_profile_clause_verification.py | 43 +++++++++++++++---- 3 files changed, 58 insertions(+), 19 deletions(-) diff --git a/docs/requirements/API-404/requirement.md b/docs/requirements/API-404/requirement.md index 552c9cfe6..e57e63713 100644 --- a/docs/requirements/API-404/requirement.md +++ b/docs/requirements/API-404/requirement.md @@ -239,7 +239,7 @@ identifies those implementation gaps and the retained canonical requirements. - DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1185-api-404-profile-alignment-preflight.md` (CP-11 profile and evidence boundaries) - TESTS → TEST `implementations/python/tests/test_issue_1185_api_404_profile_alignment.py` (Clause matrix, profile catalog, nonclaims, public documentation, and local-evidence drift checks) - DOCUMENTS → GITHUB_ISSUE `1435` (API-404 clause-to-profile evidence map and run binding on every profile) -- TESTS → TEST `implementations/python/tests/test_issue_1435_profile_clause_verification.py` (Out-of-run requests refused without record, evaluator effect, audit, or revision change on P0, P1, and P2; P0 run rebinding refused; clause-map profiles checked against profile declarations; citation and link drift checks) +- TESTS → TEST `implementations/python/tests/test_issue_1435_profile_clause_verification.py` (Out-of-run evaluation plans refused on P0, P1, and P2 without record, evaluator effect, or revision change, and without audit unless P2's planner-authorization check refuses an unregistered plan with operations first; P0 run rebinding refused; clause-map profiles checked against profile declarations; citation and link drift checks) - DOCUMENTS → GITHUB_ISSUE `1186` (CP-12: Recovery runbook and operator tooling) - DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1186-control-plane-recovery-operations-preflight.md` (CP-12 recovery, maintenance, health, and disclosure boundaries) - DOCUMENTS → DOCUMENTATION `docs/explain/sdl/control-plane-operations.md` (Operator recovery, health, shutdown, backup, restore, and upgrade runbook) diff --git a/docs/research/runtime-control-plane/conformance.md b/docs/research/runtime-control-plane/conformance.md index 43e45b065..de1d226da 100644 --- a/docs/research/runtime-control-plane/conformance.md +++ b/docs/research/runtime-control-plane/conformance.md @@ -81,8 +81,8 @@ relative to `implementations/python/packages/raes_runtime/`, and test paths to | Clause | Profiles | Implementation | Conformance cases | Operator guidance | | --- | --- | --- | --- | --- | -| `API-404-C1` | P0, P1, P2 | `control_plane.py`, `control_plane_operation_context.py`, `control_plane_admission.py`, `control_plane_mutation.py`, `control_plane_execution.py`, `control_plane_store.py`, `control_plane_store_memory.py` | `test_issue_1187_control_plane_profiles.py`, `test_issue_1187_control_plane_lifecycle_properties.py`, `test_issue_1184_atomic_idempotency_claims.py`, `test_issue_1180_snapshot_revision_cas.py`, `test_issue_1181_unified_control_plane_mutations.py`, `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1435_profile_clause_verification.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md#control-plane-operating-profiles) | -| `API-404-C2` | P1, P2 | `control_plane_store_local.py`, `control_plane_store_local_scope.py`, `control_plane_store_lease.py`, `control_plane_durability.py`, `control_plane_recovery.py` | `test_issue_1187_control_plane_process_loss.py`, `test_issue_1187_control_plane_durable_carriers.py`, `test_issue_1179_startup_reconciliation.py`, `test_issue_1183_store_ownership_leases.py`, `test_issue_1092_control_plane_crash_consistency.py`, `test_issue_1187_control_plane_profiles.py` | [Control-plane operations](../../explain/sdl/control-plane-operations.md) | +| `API-404-C1` | P0, P1, P2 | `control_plane.py`, `control_plane_configuration.py`, `control_plane_operation_context.py`, `control_plane_admission.py`, `control_plane_mutation.py`, `control_plane_execution.py`, `control_plane_store.py`, `control_plane_store_revision.py`, `control_plane_store_memory.py`, `control_plane_store_local_snapshot.py` | `test_issue_1187_control_plane_profiles.py`, `test_issue_1187_control_plane_lifecycle_properties.py`, `test_issue_1184_atomic_idempotency_claims.py`, `test_issue_1180_snapshot_revision_cas.py`, `test_issue_1181_unified_control_plane_mutations.py`, `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1435_profile_clause_verification.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md#control-plane-operating-profiles) | +| `API-404-C2` | P1, P2 | `control_plane_store_local.py`, `control_plane_store_local_records.py`, `control_plane_store_local_scope.py`, `control_plane_store_local_codec.py`, `control_plane_store_records.py`, `control_plane_store_lease.py`, `control_plane_durability.py`, `control_plane_recovery.py` | `test_issue_1187_control_plane_process_loss.py`, `test_issue_1187_control_plane_durable_carriers.py`, `test_issue_1179_startup_reconciliation.py`, `test_issue_1183_store_ownership_leases.py`, `test_issue_1092_control_plane_crash_consistency.py`, `test_issue_1187_control_plane_profiles.py` | [Control-plane operations](../../explain/sdl/control-plane-operations.md) | | `API-404-C3` | P2 | `control_plane_api/__init__.py`, `control_plane_api/_auth.py`, `control_plane_api/_offload.py`, `control_plane_api/_operation_routes.py`, `control_plane_api/_responses.py`, `control_plane_api_guards.py`, `control_plane_api_participant_retrieval.py`, `control_plane_security.py` | `test_issue_1187_control_plane_security_conformance.py`, `test_issue_1359_runtime_api_trust_boundary.py`, `test_issue_1093_request_rejection_offload.py`, `test_runtime_control_plane_api.py`, `test_issue_1187_control_plane_profiles.py` | [Serve the runtime control plane](../../public/guides/control-plane.md) | | `API-404-C4` | P0, P1, P2; P3 is unavailable | `control_plane_profiles.py` | `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1185_api_404_profile_alignment.py`, `test_issue_1187_control_plane_process_loss.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md#control-plane-operating-profiles) and the limits above | @@ -97,17 +97,29 @@ Each property #8 names is checked on every profile that guarantees it: (`test_issue_1435_profile_clause_verification.py::test_selected_p0_store_cannot_be_rebound_to_another_run`). A P1 or P2 store refuses both (`test_issue_1187_control_plane_profiles.py::test_profile_lease_scope_and_restore_keep_receipts_and_claims`). - On every profile, a request for another run leaves no record, effect, audit - or revision change. The same request for the admitted run adds one record, - one terminal audit and one evaluator start in the backend-effect witness, and - moves the snapshot revision from 0 to 1 + On every profile, an evaluation plan for another run that carries no + operations or observation demands leaves no record, effect, audit or revision + change. The same plan for the admitted run adds one record, one terminal + audit and one evaluator start in the backend-effect witness, and moves the + snapshot revision from 0 to 1 (`test_issue_1435_profile_clause_verification.py::test_request_for_another_run_changes_nothing`). - The refusal is not an admission denial, which creates audit only under the - [CP-9 preflight](../../decisions/issue-1187-control-plane-conformance-preflight.md). - The [CP-5 preflight](../../decisions/issue-1183-store-ownership-lease-preflight.md) + On P2, the provisioning, orchestration and evaluation routes check planner + authorization for a plan that carries operations or observation demands, + before the core's run check (`control_plane_api/_operation_routes.py`). So P2 + refuses an unregistered evaluation plan for another run that carries an + operation with 403 and the route's `planner-authorization-mismatch` denial + audit. P0 and P1 refuse it on the run check with no audit. No profile adds a + record or effect, and the revision stays 0 + (`test_issue_1435_profile_clause_verification.py::test_plan_with_operations_for_another_run_leaves_no_record_or_effect`). + The run refusal itself is not an admission denial, which the + [CP-2 preflight](../../decisions/issue-1181-unified-control-plane-mutations-preflight.md) + defines as an `accepted=False` receipt plus the bounded denial audit and no + stored operation. P0 and P1 raise `ValueError` and P2 answers 409, with no + receipt. The + [CP-5 preflight](../../decisions/issue-1183-store-ownership-lease-preflight.md) requires a request for another run to fail before claim or disclosure, and `operation_admission_context()` refuses it before building the operation - context that an admission-denial audit records. + context that the core's admission-denial audit records. - **Atomic state and audit.** Every operation kind commits its terminal state with exactly one actor-bound audit on P0 and P1 (`test_issue_1187_control_plane_lifecycle_properties.py::test_each_operation_kind_commits_one_immutable_actor_bound_terminal_cut`), diff --git a/implementations/python/tests/test_issue_1435_profile_clause_verification.py b/implementations/python/tests/test_issue_1435_profile_clause_verification.py index 21935bba7..7acac7375 100644 --- a/implementations/python/tests/test_issue_1435_profile_clause_verification.py +++ b/implementations/python/tests/test_issue_1435_profile_clause_verification.py @@ -2,9 +2,10 @@ ``docs/research/runtime-control-plane/conformance.md`` maps every API-404 clause to the landed code, conformance cases and operator guidance of the -profiles it binds. These cases cover the run binding that no profile case -exercised, and keep that map tied to its clauses, the profiles' declared -guarantees and its cited files. +profiles it binds. These cases add the run binding no profile case exercised: +a request for another run through each composition, and a selected P0 store +rebound to another run. They also keep that map tied to its clauses, the +profiles' declared guarantees and its cited files. """ from __future__ import annotations @@ -26,7 +27,7 @@ ) from raes_backend_stubs.stubs import create_stub_target from raes_contracts.plan_projection import evaluation_plan_model -from raes_contracts.planning import EvaluationPlan +from raes_contracts.planning import ChangeAction, EvaluationOp, EvaluationPlan from raes_runtime.control_plane import RuntimeControlPlane from raes_runtime.control_plane_profiles import ControlPlaneProfile, profile_declaration from raes_runtime.control_plane_store_memory import InMemoryControlPlaneStore @@ -46,6 +47,17 @@ "P1": "operation run scope does not match the admitted control-plane store scope", "P2": '409 {"detail":"operation conflict"}', } +# An unregistered plan that carries an operation. P2's route checks planner +# authorization before the core's run check, so it refuses the plan there and +# records its denial audit. P0 and P1 refuse it on the run check. +OPERATION_PLAN_REFUSALS = { + "P0": (RUN_REFUSALS["P0"], []), + "P1": (RUN_REFUSALS["P1"], []), + "P2": ( + '403 {"detail":"evaluation plan is not planner-authorized"}', + [("submit_evaluation", False, "planner-authorization-mismatch")], + ), +} # Written independently of the requirement document: the profiles each clause binds. CLAUSE_PROFILES = { "API-404-C1": {"P0", "P1", "P2"}, @@ -68,10 +80,9 @@ _NOT_IN_ANCHOR = re.compile(r"[^\w -]") -def _evaluate(harness: ProfileHarness, run_id: str) -> str: +def _evaluate(harness: ProfileHarness, plan: EvaluationPlan) -> str: """Submit one evaluation plan through the composition's own entry point.""" - plan = EvaluationPlan(run_id=run_id) if harness.client is not None: response = harness.client.post( "/operations/evaluation", @@ -91,7 +102,7 @@ def _evaluate(harness: ProfileHarness, run_id: str) -> str: def test_request_for_another_run_changes_nothing(profile: str, tmp_path: Path) -> None: witness = tmp_path / "effects.jsonl" with profile_harness(profile, tmp_path) as harness: - assert _evaluate(harness, "another-run") == RUN_REFUSALS[profile] + assert _evaluate(harness, EvaluationPlan(run_id="another-run")) == RUN_REFUSALS[profile] assert harness.store.load_records() == {} assert harness.store.read_audit() == [] @@ -100,13 +111,29 @@ def test_request_for_another_run_changes_nothing(profile: str, tmp_path: Path) - # The same plan for the admitted run is accepted, and the same reads see # its record, terminal audit, evaluator start and revision. - assert _evaluate(harness, ADMITTED_RUN) == ACCEPTED + assert _evaluate(harness, EvaluationPlan(run_id=ADMITTED_RUN)) == ACCEPTED (operation_id,) = harness.store.load_records() assert len(terminal_audits(harness.store, operation_id)) == 1 assert witness_events(witness) == [{"event": "evaluate", "operation_id": operation_id}] assert harness.snapshot_cut()[1] == 1 +@pytest.mark.parametrize("profile", PROFILES) +def test_plan_with_operations_for_another_run_leaves_no_record_or_effect(profile: str, tmp_path: Path) -> None: + operation = EvaluationOp( + action=ChangeAction.CREATE, address="evaluation.assertion.test", resource_type="assertion", payload={} + ) + refusal, audit = OPERATION_PLAN_REFUSALS[profile] + with profile_harness(profile, tmp_path) as harness: + assert _evaluate(harness, EvaluationPlan(run_id="another-run", operations=[operation])) == refusal + + # Read the audit before the snapshot: P2 audits the snapshot read too. + assert [(event.action, event.allowed, event.reason) for event in harness.store.read_audit()] == audit + assert harness.store.load_records() == {} + assert witness_events(tmp_path / "effects.jsonl") == [] + assert harness.snapshot_cut() == ({}, 0) + + def test_selected_p0_store_cannot_be_rebound_to_another_run() -> None: store = InMemoryControlPlaneStore() RuntimeControlPlane(create_stub_target(), store=store, run_scope=RUN_SCOPE, profile=ControlPlaneProfile.P0).close()