diff --git a/docs/requirements/API-404/requirement.md b/docs/requirements/API-404/requirement.md index 4b53c8a7ef..3722981c4a 100644 --- a/docs/requirements/API-404/requirement.md +++ b/docs/requirements/API-404/requirement.md @@ -263,7 +263,7 @@ policy choices, recovery and composition limits. - DOCUMENTS → GITHUB_ISSUE `1188` (CP-8: Served profile alignment) - DOCUMENTS → GITHUB_ISSUE `1187` (CP-9: Crash and profile conformance suite) - DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1187-control-plane-conformance-preflight.md` (CP-9 profile, fault-evidence, and security boundaries) -- DOCUMENTS → DOCUMENTATION `docs/research/runtime-control-plane/conformance.md` (Unified suite invocation, finite evidence map, and explicit profile nonclaims) +- DOCUMENTS → DOCUMENTATION `docs/research/runtime-control-plane/conformance.md` (Unified suite invocation, finite evidence map, API-404 clause-to-profile verification map, and explicit profile nonclaims) - IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/backend_result_diagnostics.py` (Value-free backend failure diagnostics without provider type names) - IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/observation_execution.py` (Observation-adapter failure diagnostics without provider exception names or text) - TESTS → TEST `implementations/python/tests/test_issue_1212_observation_demand.py` (Observation failures preserve terminal state and redact exception details from operation carriers) @@ -288,6 +288,8 @@ policy choices, recovery and composition limits. - DOCUMENTS → GITHUB_ISSUE `1185` (CP-11: API-404 requirement update) - DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1185-api-404-profile-alignment-preflight.md` (CP-11 profile and evidence boundaries) - TESTS → TEST `implementations/python/tests/test_issue_1185_api_404_profile_alignment.py` (Clause matrix, profile catalog, nonclaims, public documentation, and local-evidence drift checks) +- DOCUMENTS → GITHUB_ISSUE `1435` (API-404 clause-to-profile evidence map and run binding on every profile) +- TESTS → TEST `implementations/python/tests/test_issue_1435_profile_clause_verification.py` (Out-of-run evaluation plans refused on P0, P1, and P2 without record, evaluator effect, or revision change, and without audit unless P2's planner-authorization check refuses an unregistered plan with operations first; P0 run rebinding refused; selected P1 over the PostgreSQL ledger refuses another scope and a second owner, and ledger claims stay with their actor; clause-map profiles checked against profile declarations; clause-tagged trace, citation and link drift checks) - DOCUMENTS → GITHUB_ISSUE `1186` (CP-12: Recovery runbook and operator tooling) - DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1186-control-plane-recovery-operations-preflight.md` (CP-12 recovery, maintenance, health, and disclosure boundaries) - DOCUMENTS → DOCUMENTATION `docs/explain/sdl/control-plane-operations.md` (Operator recovery, health, shutdown, backup, restore, and upgrade runbook) diff --git a/docs/research/runtime-control-plane/conformance.md b/docs/research/runtime-control-plane/conformance.md index 99f77e8530..b5e56015cd 100644 --- a/docs/research/runtime-control-plane/conformance.md +++ b/docs/research/runtime-control-plane/conformance.md @@ -69,3 +69,125 @@ external-effect, tamper-evidence, or universal recovery-proof claim. Payload digests detect corruption; a writer able to replace both content and digest is not thereby authenticated. ASGI tests do not verify deployment TLS, trusted-proxy header stripping, external backend recovery, or a matching backend backup. + +## API-404 clause verification + +[API-404](../../requirements/API-404/requirement.md) states the clauses and +the profiles each clause binds. `profile_declaration()` holds each profile's +guarantee and nonclaim identifiers. The table maps each clause to the landed +code, conformance cases, and operator guidance behind it. Code paths are +relative to `implementations/python/packages/raes_runtime/`, and test paths to +`implementations/python/tests/`. + +A P1 core, and so the P2 server over it, can select one of two durable stores: +the SQLite store (`control_plane_store_local.py`) or the PostgreSQL ledger +(`control_plane_store_postgresql.py`). Both declare every store capability +that P1 requires. The rows also cite the supervised-execution path that +issue #1362 added. Each row holds every `raes_runtime` code file, test and +document that an API-404 trace line names for that clause. + +| Clause | Profiles | Implementation | Conformance cases | Operator guidance | +| --- | --- | --- | --- | --- | +| `API-404-C1` | P0, P1, P2 | `control_plane.py`, `control_plane_configuration.py`, `control_plane_operation_context.py`, `control_plane_admission.py`, `control_plane_mutation.py`, `control_plane_execution.py`, `control_plane_store.py`, `control_plane_store_revision.py`, `control_plane_store_memory.py`, `control_plane_store_local_snapshot.py`, `control_plane_store_postgresql.py`, `control_plane_store_postgresql_mutations.py`, `control_plane_supervised_execution.py`, `control_plane_supervision.py`, `operation_invocation_ledger.py`, `operation_transcript_policy.py`, `operation_execution_adapter.py`, `operation_command_commitment.py`, `operation_result_finalization.py`, `operation_reconciliation.py`, `operation_worker_authority.py`, `operation_temporal_worker.py`, `runtime_resource_ownership.py`, `manager_lifecycle.py` | `test_issue_1187_control_plane_profiles.py`, `test_issue_1187_control_plane_lifecycle_properties.py`, `test_issue_1184_atomic_idempotency_claims.py`, `test_issue_1180_snapshot_revision_cas.py`, `test_issue_1181_unified_control_plane_mutations.py`, `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1435_profile_clause_verification.py`, `test_issue_1362_postgresql_store.py`, `test_issue_1362_state_authority.py`, `test_issue_1362_runtime_supervision.py`, `test_issue_1362_bounded_execution.py`, `test_issue_1362_invocation_ledger.py`, `test_issue_1362_provider_execution.py`, `test_issue_1362_reconciliation.py`, `test_issue_1362_distributed_authority.py`, `test_issue_1362_commit_authority.py`, `test_issue_1362_temporal_worker.py`, `test_issue_1362_runtime_imports.py`, `test_issue_1362_review_regressions.py`, `test_issue_1362_control_budget.py`, `test_issue_1362_review_round_two.py`, `test_issue_1362_command_privacy.py`, `test_issue_1362_observation_retention.py`, `test_issue_1362_distributed_integration.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md#control-plane-operating-profiles) | +| `API-404-C2` | P1, P2 | `control_plane_store_local.py`, `control_plane_store_local_records.py`, `control_plane_store_local_scope.py`, `control_plane_store_local_codec.py`, `control_plane_store_records.py`, `control_plane_store_lease.py`, `control_plane_durability.py`, `control_plane_recovery.py`, `control_plane_store_postgresql.py`, `control_plane_store_postgresql_mutations.py`, `control_plane_store_postgresql_schema.py`, `control_plane_store_local_invocations.py`, `control_plane_supervised_execution.py`, `operation_invocation_ledger.py`, `operation_transcript_policy.py`, `operation_command_commitment.py`, `operation_result_finalization.py`, `operation_reconciliation.py`, `operation_worker_authority.py`, `operation_temporal.py`, `operation_temporal_worker.py` | `test_issue_1187_control_plane_process_loss.py`, `test_issue_1187_control_plane_durable_carriers.py`, `test_issue_1179_startup_reconciliation.py`, `test_issue_1183_store_ownership_leases.py`, `test_issue_1092_control_plane_crash_consistency.py`, `test_issue_1187_control_plane_profiles.py`, `test_issue_1362_postgresql_store.py`, `test_issue_1362_invocation_ledger.py`, `test_issue_1362_reconciliation.py`, `test_issue_1362_distributed_authority.py`, `test_issue_1362_lost_ack.py`, `test_issue_1362_temporal_worker.py`, `test_issue_1362_review_regressions.py`, `test_issue_1362_control_budget.py`, `test_issue_1362_review_round_two.py`, `test_issue_1362_command_privacy.py`, `test_issue_1362_observation_retention.py`, `test_issue_1362_distributed_integration.py`, `test_issue_1435_profile_clause_verification.py` | [Control-plane operations](../../explain/sdl/control-plane-operations.md), [Supervised execution](../../public/guides/supervised-execution.md) | +| `API-404-C3` | P2 | `control_plane_api/__init__.py`, `control_plane_api/_auth.py`, `control_plane_api/_offload.py`, `control_plane_api/_operation_routes.py`, `control_plane_api/_responses.py`, `control_plane_api_guards.py`, `control_plane_api_participant_retrieval.py`, `control_plane_security.py`, `control_plane_api/_supervision_routes.py`, `control_plane_supervision.py`, `operation_execution_adapter.py` | `test_issue_1187_control_plane_security_conformance.py`, `test_issue_1359_runtime_api_trust_boundary.py`, `test_issue_1093_request_rejection_offload.py`, `test_runtime_control_plane_api.py`, `test_issue_1187_control_plane_profiles.py`, `test_issue_1362_runtime_supervision.py`, `test_issue_1362_supervision_http.py`, `test_issue_1362_http_capacity.py`, `test_issue_1362_review_regressions.py`, `test_issue_1362_review_round_two.py`, `test_issue_1362_distributed_integration.py` | [Serve the runtime control plane](../../public/guides/control-plane.md) | +| `API-404-C4` | P0, P1, P2; P3 is unavailable | `control_plane_profiles.py` | `test_issue_1189_control_plane_profile_declarations.py`, `test_issue_1185_api_404_profile_alignment.py`, `test_issue_1187_control_plane_process_loss.py` | [Control-plane operating profiles](../../explain/sdl/runtime-architecture.md#control-plane-operating-profiles), [Supervised execution](../../public/guides/supervised-execution.md) and the limits above | + +Each property #8 names is checked on every profile that guarantees it: + +- **Actor binding.** Retry, readback and disclosure stay with the submitting + actor on P0, and on P1 and P2 with the SQLite store + (`test_issue_1187_control_plane_profiles.py::test_profile_scoped_retry_status_audit_and_cache_coherence`). + The PostgreSQL ledger finds a retained claim only for the actor that made it + (`test_issue_1435_profile_clause_verification.py::test_postgresql_ledger_claims_stay_with_the_submitting_actor`). +- **Target and run binding.** A selected P0 store refuses another target + (`test_issue_1189_control_plane_profile_declarations.py::test_selected_p0_uses_canonical_declaration_and_binds_one_scope`) + and another run + (`test_issue_1435_profile_clause_verification.py::test_selected_p0_store_cannot_be_rebound_to_another_run`). + A P1 or P2 SQLite store refuses both + (`test_issue_1187_control_plane_profiles.py::test_profile_lease_scope_and_restore_keep_receipts_and_claims`). + A selected P1 core refuses a PostgreSQL ledger bound to another target or + run at admission, before it reads an operation or changes the ledger + (`test_issue_1435_profile_clause_verification.py::test_selected_p1_refuses_a_postgresql_ledger_for_another_scope`). + The ledger also refuses a record for another target or run before it writes + (`test_issue_1362_postgresql_store.py::test_postgresql_mutation_admits_exact_record_scope_before_writing`). + On every profile, an evaluation plan for another run that carries no + operations or observation demands leaves no record, effect, audit or revision + change. The same plan for the admitted run adds one record, one terminal + audit and one evaluator start in the backend-effect witness, and moves the + snapshot revision from 0 to 1 + (`test_issue_1435_profile_clause_verification.py::test_request_for_another_run_changes_nothing`). + On P2, the provisioning, orchestration and evaluation routes check planner + authorization for a plan that carries operations or observation demands, + before the core's run check (`control_plane_api/_operation_routes.py`). So P2 + refuses an unregistered evaluation plan for another run that carries an + operation with 403 and the route's `planner-authorization-mismatch` denial + audit. P0 and P1 refuse it on the run check with no audit. No profile adds a + record or effect, and the revision stays 0 + (`test_issue_1435_profile_clause_verification.py::test_plan_with_operations_for_another_run_leaves_no_record_or_effect`). + The run refusal itself is not an admission denial, which the + [CP-2 preflight](../../decisions/issue-1181-unified-control-plane-mutations-preflight.md) + defines as an `accepted=False` receipt plus the bounded denial audit and no + stored operation. P0 and P1 raise `ValueError` and P2 answers 409, with no + receipt. The + [CP-5 preflight](../../decisions/issue-1183-store-ownership-lease-preflight.md) + requires a request for another run to fail before claim or disclosure, and + `operation_admission_context()` refuses it before building the operation + context that the core's admission-denial audit records. +- **Atomic state and audit.** Every operation kind commits its terminal state + with exactly one actor-bound audit on P0 and P1 + (`test_issue_1187_control_plane_lifecycle_properties.py::test_each_operation_kind_commits_one_immutable_actor_bound_terminal_cut`), + and so does an HTTP submission on P2 + (`test_issue_1187_control_plane_profiles.py::test_profile_scoped_retry_status_audit_and_cache_coherence`). + With the PostgreSQL ledger, an operation delivered through the HTTP server + keeps exactly one terminal audit after a duplicate delivery + (`test_issue_1362_distributed_integration.py::test_real_delivery_control_and_reopening_preserve_the_consumed_fence`). +- **Retained idempotency.** P1 and P2 with the SQLite store keep receipts and + claims across restore + (`test_issue_1187_control_plane_profiles.py::test_profile_lease_scope_and_restore_keep_receipts_and_claims`). + The PostgreSQL ledger returns the original receipt for a retry after + process loss + (`test_issue_1362_distributed_integration.py::test_actual_process_loss_never_reissues_a_consumed_grant`). + P0 loses them with its process + (`test_issue_1187_control_plane_process_loss.py::test_p0_process_loss_loses_run_and_claim_even_when_external_effect_survives`). +- **Ownership.** A P0 store has one live owner + (`test_issue_1189_control_plane_profile_declarations.py::test_selected_p0_store_has_one_active_owner_and_releases_it_on_close`). + P1 and P2 admit one SQLite lease owner and refuse a second + (`test_issue_1187_control_plane_profiles.py::test_profile_lease_scope_and_restore_keep_receipts_and_claims`). + A selected P1 core refuses a PostgreSQL ledger that another session owns, + and one ledger object admits only one runtime + (`test_issue_1435_profile_clause_verification.py::test_selected_p1_admits_one_postgresql_ledger_owner`). +- **Revision checks.** A stale snapshot write fails on every profile + (`test_issue_1187_control_plane_profiles.py::test_profile_scoped_retry_status_audit_and_cache_coherence`). + The PostgreSQL ledger refuses a stale terminal commit before it writes + (`test_issue_1362_postgresql_store.py::test_postgresql_terminal_boundary_rejects_stale_revision_before_any_write`). +- **Startup classification.** P1 and P2 with the SQLite store classify + interrupted work without replay + (`test_issue_1187_control_plane_process_loss.py::test_process_loss_preserves_atomic_cut_and_classifies_without_replay`). + The PostgreSQL ledger does the same after an abrupt loss at each grant, + dispatch and terminal boundary + (`test_issue_1362_distributed_integration.py::test_actual_process_loss_never_reissues_a_consumed_grant`). + It does not apply to P0, which keeps no state across process loss. + +The two `test_issue_1362_distributed_integration.py` cases need a live +PostgreSQL database and Temporal service. They run only in the CI +`runtime-supervision` job and the canonical integration lane, which set +`RAES_EXECUTION_INTEGRATION=1`. They compose the ledger without a selected +profile. The `test_issue_1435_profile_clause_verification.py` ledger cases +select P1 and use an in-memory stand-in for the database connection. + +P2 serves a selected P1 core and adds no store of its own. So its C1 and C2 +evidence is the evidence for the P1 core's store, SQLite or PostgreSQL, plus +the HTTP cases. +`test_issue_1435_profile_clause_verification.py` derives each clause's profiles +from `profile_declaration()`: the available profiles whose guarantees include +the clause's identifiers, or every available profile for C4. It fails if that +set, this table and an independently written expectation disagree. It also +fails if an Implementation or Conformance cell holds an entry that is not a +cited file, if the section cites a file or test that does not exist, or if a +link names a missing page or heading. An API-404 trace line that names clauses +must appear in each named row. Its code file must be in the Implementation +cell, its test in the Conformance cell, and its documentation in the Operator +guidance cell. The check reads code under `raes_runtime`, tests under +`implementations/python/tests/` and pages under `docs/`. It skips other traced +files, such as CI workflows. diff --git a/implementations/python/tests/control_plane_conformance_fixtures.py b/implementations/python/tests/control_plane_conformance_fixtures.py index 48f06a8145..1f3020b0d8 100644 --- a/implementations/python/tests/control_plane_conformance_fixtures.py +++ b/implementations/python/tests/control_plane_conformance_fixtures.py @@ -15,7 +15,7 @@ RecoveryObservationRequest, RecoveryObservationResult, ) -from raes_backend_stubs.stubs import create_stub_target +from raes_backend_stubs.stubs import StubEvaluator, create_stub_target from raes_contracts.plan_projection import provisioning_plan_model from raes_contracts.planning import ProvisioningPlan from raes_contracts.runtime_state import OperationKind, OperationState @@ -81,6 +81,18 @@ def apply(self, plan, snapshot): return result +class WitnessEvaluator(StubEvaluator): + # An evaluator start is a backend effect too, so it joins the same witness. + def __init__(self, witness: Path) -> None: + super().__init__() + self.witness = witness + + def start(self, plan, snapshot): + assert plan.operation_id + record_effect(self.witness, "evaluate", plan.operation_id) + return super().start(plan, snapshot) + + class WitnessObserver: def __init__(self, witness: Path) -> None: self.witness = witness @@ -109,7 +121,11 @@ def witness_target(witness: Path, *, observe: bool = False, boundary=None): # Recovery intentionally refuses to invent a provisioning plan to authorize # new realization/resource state from an observer's assertion alone. target = create_stub_target(with_realization_envelope=False) - target = replace(target, provisioner=WitnessProvisioner(target.provisioner, witness, boundary)) + target = replace( + target, + provisioner=WitnessProvisioner(target.provisioner, witness, boundary), + evaluator=WitnessEvaluator(witness), + ) if observe: capability = RecoveryObservationCapabilities( name="conformance-witness", supported_operation_kinds=frozenset({OperationKind.PROVISIONING}) diff --git a/implementations/python/tests/test_issue_1362_distributed_integration.py b/implementations/python/tests/test_issue_1362_distributed_integration.py index c876d478af..cbd4817ae0 100644 --- a/implementations/python/tests/test_issue_1362_distributed_integration.py +++ b/implementations/python/tests/test_issue_1362_distributed_integration.py @@ -138,9 +138,14 @@ async def _exercise(blocked, tmp_path): ) store = PostgreSQLControlPlaneStore(config) await asyncio.to_thread(store.initialize_schema) + # The budget runs from acceptance, so it must cover cold engine submission + # and delivery (each can take seconds) and still expire while the blocked + # start holds the backend call. + execution_ms = 15000 if blocked else 20000 provider = Provider() if blocked: provider.release.clear() + provider.block_seconds = execution_ms / 1000 + 5 cli = Path(os.environ["RAES_TEMPORAL_CLI"]).resolve(strict=True) async with await WorkflowEnvironment.start_local( dev_server_existing_path=str(cli), dev_server_database_filename=str(tmp_path / "engine-history.sqlite3") @@ -150,15 +155,19 @@ async def _exercise(blocked, tmp_path): async def connected(): return await Client.connect(address, namespace="default") + # A just-started development server can take over two seconds to + # accept its first workflow. The RPC budget and the admission budget + # that bounds engine submission both leave room for that. delivery = TemporalExecutionAdapter( - TemporalExecutionConfiguration(worker_identity="worker-1362"), client_factory=connected + TemporalExecutionConfiguration(worker_identity="worker-1362", rpc_budget_ms=10000), + client_factory=connected, ) plane = RuntimeControlPlane( target(provider), store=store, operation_evidence=Evidence(), distributed_execution=delivery, - supervision=OperationSupervisionConfiguration(execution_ms=2000 if blocked else 15000), + supervision=OperationSupervisionConfiguration(admission_ms=10000, execution_ms=execution_ms), ) listener = socket.socket() listener.bind(("127.0.0.1", 0)) @@ -203,7 +212,8 @@ async def connected(): ) await _until(lambda: provider.cancels == 1) await _until( - lambda: plane.get_operation(receipt.operation_id).state is OperationState.INDETERMINATE + lambda: plane.get_operation(receipt.operation_id).state is OperationState.INDETERMINATE, + timeout=execution_ms / 1000 + 5, ) assert store.load_records()[receipt.operation_id].invocation.reserved provider.release.set() diff --git a/implementations/python/tests/test_issue_1362_provider_execution.py b/implementations/python/tests/test_issue_1362_provider_execution.py index adcc6153e2..96e43996ee 100644 --- a/implementations/python/tests/test_issue_1362_provider_execution.py +++ b/implementations/python/tests/test_issue_1362_provider_execution.py @@ -26,6 +26,7 @@ def __init__(self): self.started = Event() self.release = Event() self.release.set() + self.block_seconds = 3 self._lock = Lock() self.sequence = 0 self.starts = 0 @@ -69,7 +70,7 @@ def start_operation(self, request): self.request = request self.starts += 1 self.started.set() - assert self.release.wait(3) + assert self.release.wait(self.block_seconds) return self.report(request, {"kind": "acknowledgement", "disposition": "accepted"}) def observe_operation(self, control): diff --git a/implementations/python/tests/test_issue_1435_profile_clause_verification.py b/implementations/python/tests/test_issue_1435_profile_clause_verification.py new file mode 100644 index 0000000000..3272ca3fa0 --- /dev/null +++ b/implementations/python/tests/test_issue_1435_profile_clause_verification.py @@ -0,0 +1,418 @@ +"""API-404 clause verification against each control-plane profile (#1435, for #8). + +``docs/research/runtime-control-plane/conformance.md`` maps every API-404 +clause to the landed code, conformance cases and operator guidance of the +profiles it binds. These cases add the run binding no profile case exercised: +a request for another run through each composition, and a selected P0 store +rebound to another run. They add the actor, scope and owner bindings of a +selected P1 core over the PostgreSQL ledger, through an in-memory stand-in for +its database connection. They also keep that map tied to its clauses, the +profiles' declared guarantees, the requirement's clause-tagged trace lines and +its cited files. +""" + +from __future__ import annotations + +import ast +import re +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path + +import pytest +from control_plane_conformance_fixtures import ( + KEY, + PROFILES, + RUN_SCOPE, + ProfileHarness, + identities, + profile_harness, + terminal_audits, + witness_events, +) +from raes_backend_stubs.stubs import create_stub_target +from raes_contracts.plan_projection import evaluation_plan_model +from raes_contracts.planning import ChangeAction, EvaluationOp, EvaluationPlan +from raes_runtime import control_plane_store_postgresql_mutations as ledger_mutations +from raes_runtime.control_plane import RuntimeControlPlane +from raes_runtime.control_plane_operation_context import runtime_target_scope +from raes_runtime.control_plane_profiles import ControlPlaneProfile, profile_declaration +from raes_runtime.control_plane_store_memory import InMemoryControlPlaneStore +from raes_runtime.control_plane_store_postgresql import PostgreSQLControlPlaneStore, PostgreSQLStoreConfiguration +from raes_runtime.control_plane_store_postgresql_schema import POSTGRES_LEDGER_SCHEMA_VERSION +from test_issue_1181_unified_control_plane_mutations import _running_record +from test_issue_1185_api_404_profile_alignment import ( + COMMON_GUARANTEES, + DURABLE_GUARANTEES, + REQUIREMENT_PATH, + TRANSPORT_GUARANTEES, +) + +pytestmark = pytest.mark.control_plane_conformance + +ROOT = Path(__file__).resolve().parents[3] +CONFORMANCE_PAGE = ROOT / "docs/research/runtime-control-plane/conformance.md" +CODE_ROOT = ROOT / "implementations/python/packages/raes_runtime" +TEST_ROOT = Path(__file__).resolve().parent +ADMITTED_RUN = RUN_SCOPE.removeprefix("run:") +ACCEPTED = "accepted" +# How each composition refuses a request whose run differs from its admitted run. +RUN_REFUSALS = { + "P0": "operation run scope does not match the admitted control-plane store scope", + "P1": "operation run scope does not match the admitted control-plane store scope", + "P2": '409 {"detail":"operation conflict"}', +} +# An unregistered plan that carries an operation. P2's route checks planner +# authorization before the core's run check, so it refuses the plan there and +# records its denial audit. P0 and P1 refuse it on the run check. +OPERATION_PLAN_REFUSALS = { + "P0": (RUN_REFUSALS["P0"], []), + "P1": (RUN_REFUSALS["P1"], []), + "P2": ( + '403 {"detail":"evaluation plan is not planner-authorized"}', + [("submit_evaluation", False, "planner-authorization-mismatch")], + ), +} +# Written independently of the requirement document: the profiles each clause binds. +CLAUSE_PROFILES = { + "API-404-C1": {"P0", "P1", "P2"}, + "API-404-C2": {"P1", "P2"}, + "API-404-C3": {"P2"}, + "API-404-C4": {"P0", "P1", "P2"}, +} +# The guarantee identifiers API-404 names for each clause, as the #1185 module +# pins them. C4 names none, so it binds every available profile. +CLAUSE_GUARANTEES = { + "API-404-C1": COMMON_GUARANTEES, + "API-404-C2": DURABLE_GUARANTEES, + "API-404-C3": TRANSPORT_GUARANTEES, + "API-404-C4": set(), +} +_CITED_FILE = re.compile(r"`([\w/]+\.py)(?:::(\w+))?`") +# A link's page, empty for this page, and its optional heading fragment. +_LINK_TARGET = re.compile(r"\]\(([^)#\s]*)(?:#([^)\s]*))?\)") +_HEADING = re.compile(r"^#+ (.+)$", re.MULTILINE) +_NOT_IN_ANCHOR = re.compile(r"[^\w -]") +# A trace line that names its clauses, such as "(API-404-C1/C3 ...". +_CLAUSE_TRACE = re.compile( + r"^- (?:IMPLEMENTS → CODE_FILE|TESTS → TEST|DOCUMENTS → DOCUMENTATION) `([^`]+)` \(API-404-(C\d(?:/C\d)*)\b", + re.MULTILINE, +) +# The table cell that holds each kind of traced artifact, and its path prefix. +_TRACE_CELLS = ( + ("implementations/python/packages/raes_runtime/", 2), + ("implementations/python/tests/", 3), + ("docs/", 4), +) + + +def _evaluate(harness: ProfileHarness, plan: EvaluationPlan) -> str: + """Submit one evaluation plan through the composition's own entry point.""" + + if harness.client is not None: + response = harness.client.post( + "/operations/evaluation", + json=evaluation_plan_model(plan).model_dump(mode="json"), + headers=harness.headers(), + ) + accepted = response.status_code == 200 and response.json()["accepted"] + return ACCEPTED if accepted else f"{response.status_code} {response.text}" + try: + receipt = harness.plane.submit_evaluation(plan, idempotency_key=KEY, identity=identities()["alice"]) + except ValueError as refusal: + return str(refusal) + return ACCEPTED if receipt.accepted else repr(receipt.diagnostics) + + +@pytest.mark.parametrize("profile", PROFILES) +def test_request_for_another_run_changes_nothing(profile: str, tmp_path: Path) -> None: + witness = tmp_path / "effects.jsonl" + with profile_harness(profile, tmp_path) as harness: + assert _evaluate(harness, EvaluationPlan(run_id="another-run")) == RUN_REFUSALS[profile] + + assert harness.store.load_records() == {} + assert harness.store.read_audit() == [] + assert witness_events(witness) == [] + assert harness.snapshot_cut() == ({}, 0) + + # The same plan for the admitted run is accepted, and the same reads see + # its record, terminal audit, evaluator start and revision. + assert _evaluate(harness, EvaluationPlan(run_id=ADMITTED_RUN)) == ACCEPTED + (operation_id,) = harness.store.load_records() + assert len(terminal_audits(harness.store, operation_id)) == 1 + assert witness_events(witness) == [{"event": "evaluate", "operation_id": operation_id}] + assert harness.snapshot_cut()[1] == 1 + + +@pytest.mark.parametrize("profile", PROFILES) +def test_plan_with_operations_for_another_run_leaves_no_record_or_effect(profile: str, tmp_path: Path) -> None: + operation = EvaluationOp( + action=ChangeAction.CREATE, address="evaluation.assertion.test", resource_type="assertion", payload={} + ) + refusal, audit = OPERATION_PLAN_REFUSALS[profile] + with profile_harness(profile, tmp_path) as harness: + assert _evaluate(harness, EvaluationPlan(run_id="another-run", operations=[operation])) == refusal + + # Read the audit before the snapshot: P2 audits the snapshot read too. + assert [(event.action, event.allowed, event.reason) for event in harness.store.read_audit()] == audit + assert harness.store.load_records() == {} + assert witness_events(tmp_path / "effects.jsonl") == [] + assert harness.snapshot_cut() == ({}, 0) + + +def test_selected_p0_store_cannot_be_rebound_to_another_run() -> None: + store = InMemoryControlPlaneStore() + RuntimeControlPlane(create_stub_target(), store=store, run_scope=RUN_SCOPE, profile=ControlPlaneProfile.P0).close() + target = create_stub_target() + + with pytest.raises(ValueError, match="store scope does not match runtime admission"): + RuntimeControlPlane(target, store=store, run_scope="run:another", profile=ControlPlaneProfile.P0) + + # The refusal leaves the store bound to its first scope, which reopens. + RuntimeControlPlane(create_stub_target(), store=store, run_scope=RUN_SCOPE, profile=ControlPlaneProfile.P0).close() + + +class _Rows: + def __init__(self, rows: list[tuple[object, ...]]) -> None: + self._rows = rows + + def fetchone(self) -> tuple[object, ...] | None: + return self._rows[0] if self._rows else None + + def fetchall(self) -> list[tuple[object, ...]]: + return self._rows + + +class _LedgerDatabase: + """The ledger metadata and owner lock that runtime admission reads and writes.""" + + def __init__(self, metadata: dict[str, str]) -> None: + self.metadata = dict(metadata) + self.lock_holder: object | None = None + self.connections: list[_LedgerConnection] = [] + + def connect(self, *, autocommit: bool = False) -> _LedgerConnection: + connection = _LedgerConnection(self) + self.connections.append(connection) + return connection + + +class _LedgerConnection: + """Answers only the statements of ``PostgreSQLControlPlaneStore.admit_runtime``.""" + + def __init__(self, database: _LedgerDatabase) -> None: + self.database = database + self.closed = False + self._pending: dict[str, str] | None = None + + def __enter__(self) -> _LedgerConnection: + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + def close(self) -> None: + if self.database.lock_holder is self: + self.database.lock_holder = None + self.closed = True + + @contextmanager + def transaction(self): + # Metadata changes land only when the transaction body completes. + self._pending = dict(self.database.metadata) + try: + yield + self.database.metadata = self._pending + finally: + self._pending = None + + @contextmanager + def cursor(self): + yield self + + def executemany(self, statement: str, rows) -> None: + assert statement.startswith("INSERT INTO raes_ledger_metadata") + self._pending.update(rows) + + def execute(self, statement: str, params: tuple[object, ...] = ()) -> _Rows: + if statement.startswith("SELECT pg_try_advisory_lock"): + if self.database.lock_holder is None: + self.database.lock_holder = self + return _Rows([(self.database.lock_holder is self,)]) + if statement.startswith("SELECT EXISTS(SELECT 1 FROM pg_locks"): + return _Rows([(self.database.lock_holder is self,)]) + if statement.startswith("SELECT key, value FROM raes_ledger_metadata"): + return _Rows(sorted(self._pending.items())) + if statement.startswith("INSERT INTO raes_ledger_metadata(key, value) VALUES ('owner-generation'"): + self._pending["owner-generation"] = params[0] + return _Rows([]) + raise AssertionError(f"unexpected ledger statement: {statement}") + + +def _ledger(database: _LedgerDatabase, monkeypatch: pytest.MonkeyPatch) -> PostgreSQLControlPlaneStore: + ledger = PostgreSQLControlPlaneStore( + PostgreSQLStoreConfiguration(host="ledger.example", database="rae_ledger", user="rae_ledger") + ) + monkeypatch.setattr(ledger, "_connect", database.connect) + return ledger + + +def _scoped_ledger_metadata() -> dict[str, str]: + return { + "schema-version": POSTGRES_LEDGER_SCHEMA_VERSION, + "target-scope": runtime_target_scope(create_stub_target().name), + "run-scope": RUN_SCOPE, + "owner-generation": "1", + } + + +@pytest.mark.parametrize(("target_name", "run"), [("other", RUN_SCOPE), ("stub", "run:another")]) +def test_selected_p1_refuses_a_postgresql_ledger_for_another_scope( + target_name: str, run: str, monkeypatch: pytest.MonkeyPatch +) -> None: + database = _LedgerDatabase(_scoped_ledger_metadata()) + ledger = _ledger(database, monkeypatch) + target = replace(create_stub_target(), name=target_name) + + # P1 selection accepts the ledger, so the refusal comes from its admission. + with pytest.raises(RuntimeError, match="PostgreSQL ledger scope does not match runtime admission"): + RuntimeControlPlane(target, store=ledger, run_scope=run, profile=ControlPlaneProfile.P1) + + assert database.metadata == _scoped_ledger_metadata() + assert database.lock_holder is None + assert all(connection.closed for connection in database.connections) + assert not ledger.admitted + + # The ledger's own scope is still admitted, and admission advances its owner generation. + ledger.admit_runtime(target_scope=runtime_target_scope("stub"), run_scope=RUN_SCOPE) + assert ledger.admitted + assert database.metadata["owner-generation"] == "2" + + +def test_selected_p1_admits_one_postgresql_ledger_owner(monkeypatch: pytest.MonkeyPatch) -> None: + database = _LedgerDatabase(_scoped_ledger_metadata()) + database.lock_holder = object() + ledger = _ledger(database, monkeypatch) + target = create_stub_target() + + with pytest.raises(RuntimeError, match="PostgreSQL ledger already has a runtime owner"): + RuntimeControlPlane(target, store=ledger, run_scope=RUN_SCOPE, profile=ControlPlaneProfile.P1) + assert database.metadata == _scoped_ledger_metadata() + assert all(connection.closed for connection in database.connections) + assert not ledger.admitted + + database.lock_holder = None + target_scope = runtime_target_scope("stub") + ledger.admit_runtime(target_scope=target_scope, run_scope=RUN_SCOPE) + with pytest.raises(RuntimeError, match="PostgreSQL ledger already has runtime admission"): + ledger.admit_runtime(target_scope=target_scope, run_scope=RUN_SCOPE) + assert database.metadata["owner-generation"] == "2" + + +def test_postgresql_ledger_claims_stay_with_the_submitting_actor() -> None: + record = _running_record("ledger-actor") + context = record.receipt.context + other_actor = context.model_copy(update={"actor_id": f"{context.actor_id}-other"}) + + class Ledger: + def _load_records(self, connection): + return {record.receipt.operation_id: record} + + key = record.idempotency_key + assert ledger_mutations.find_by_idempotency(Ledger(), None, key, context=context) == record + assert ledger_mutations.find_by_idempotency(Ledger(), None, key, context=other_actor) is None + + +def _clause_section() -> str: + page = CONFORMANCE_PAGE.read_text(encoding="utf-8") + return page.split("\n## API-404 clause verification\n", 1)[1].split("\n## ", 1)[0] + + +def _clause_rows() -> dict[str, list[str]]: + rows = [line.strip().strip("|").split("|") for line in _clause_section().splitlines() if line.startswith("| `API")] + return {cells[0].strip().strip("`"): [cell.strip() for cell in cells] for cells in rows} + + +def _declared_profiles(guarantees: set[str]) -> set[str]: + """Available profiles whose declared guarantees include every given identifier.""" + + declarations = [profile_declaration(profile) for profile in ControlPlaneProfile] + return { + declaration.profile.value + for declaration in declarations + if declaration.available and guarantees <= {claim.identifier for claim in declaration.guarantees} + } + + +def test_clause_map_binds_each_clause_to_its_declared_profiles() -> None: + declared = {clause: _declared_profiles(guarantees) for clause, guarantees in CLAUSE_GUARANTEES.items()} + # A note after ";" in a cell, such as "P3 is unavailable", binds no profile. + table = { + clause: set(re.findall(r"\bP\d\b", cells[1].partition(";")[0])) for clause, cells in _clause_rows().items() + } + + assert declared == CLAUSE_PROFILES + assert table == CLAUSE_PROFILES + + +def _cites_existing(path: str, function: str) -> bool: + source = (TEST_ROOT if Path(path).name.startswith("test_") else CODE_ROOT) / path + if not source.is_file(): + return False + tree = ast.parse(source.read_text(encoding="utf-8")) + return not function or function in {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} + + +def test_clause_map_cites_existing_code_and_tests() -> None: + rows = _clause_rows() + # Every entry of each Implementation and Conformance cell must be a citation + # the pattern reads, so an emptied cell or a narrowed pattern fails here. + entries = [entry for cells in rows.values() for cell in (cells[2], cells[3]) for entry in cell.split(", ")] + citations = sorted(set(_CITED_FILE.findall(_clause_section()))) + + assert rows.keys() == CLAUSE_PROFILES.keys() + assert [entry for entry in entries if not _CITED_FILE.fullmatch(entry)] == [] + assert [citation for citation in citations if not _cites_existing(*citation)] == [] + + +def _heading_anchors(page: Path) -> set[str]: + """The fragments GitHub derives from a page's headings, without duplicate suffixes.""" + + headings = _HEADING.findall(page.read_text(encoding="utf-8")) + return {_NOT_IN_ANCHOR.sub("", heading.strip().lower()).replace(" ", "-") for heading in headings} + + +def _link_resolves(target: str, fragment: str) -> bool: + page = CONFORMANCE_PAGE.parent / target if target else CONFORMANCE_PAGE + return page.is_file() and (not fragment or fragment in _heading_anchors(page)) + + +def test_clause_map_links_resolve() -> None: + links = _LINK_TARGET.findall(_clause_section()) + + assert links + assert [link for link in links if not _link_resolves(*link)] == [] + + +def test_clause_map_cites_each_clause_tagged_trace_line() -> None: + rows = _clause_rows() + traced = _CLAUSE_TRACE.findall(REQUIREMENT_PATH.read_text(encoding="utf-8")) + missing = [] + for path, clauses in traced: + for prefix, cell in _TRACE_CELLS: + if not path.startswith(prefix): + continue + for clause in clauses.split("/"): + cells = rows[f"API-404-{clause}"] + if prefix == "docs/": + pages = { + (CONFORMANCE_PAGE.parent / target).resolve() for target, _ in _LINK_TARGET.findall(cells[cell]) + } + cited = (ROOT / path).resolve() in pages + else: + cited = path.removeprefix(prefix) in {file for file, _ in _CITED_FILE.findall(cells[cell])} + if not cited: + missing.append((clause, path)) + + assert traced + assert missing == []