diff --git a/core/application/__init__.py b/core/application/__init__.py index 80a125d..1e4a17b 100644 --- a/core/application/__init__.py +++ b/core/application/__init__.py @@ -4,7 +4,13 @@ (health checks), Doc 03 §15 and §24 (orchestration; experiment scope policy stays out of core domain logic), Doc 02 §4 (layering). Built in: P0-close (configuration and health), P4 (``VerificationScopePolicy``), -P6b (``EvaluationOrchestrator``). +P6c (the evaluation flow: ``orchestrator.EvaluationOrchestrator``, ``baseline.establish_baseline``, +``recorder.StageRecorder``, ``chain.describe_chain``, ``ids``). + +The orchestrator coordinates only (Doc 03 §15): it runs the stages in order, records the evidence +and the audit events of each, and leaves every decision to ``core.promotion``. It reaches the +database through a factory that opens a unit of work and never imports ``sqlite3`` or +``core.persistence`` (the contract test of P6c step 1 pins it). Import restrictions (tests/contract/test_architecture_boundaries.py): nothing in ``core.*`` outside ``core.application`` may import this package (R2); diff --git a/core/application/baseline.py b/core/application/baseline.py new file mode 100644 index 0000000..9a2ee5f --- /dev/null +++ b/core/application/baseline.py @@ -0,0 +1,419 @@ +"""Baseline establishment: the first Trusted State of a lineage (P6c step 3; Doc 06 §24, Doc 13 §13; +Doc 11 §8, §44; C-30, C-32, C-59, C-171, C-176). + +Doc 06 §24: before repair begins the system establishes an initial Trusted State and its protected +invariants. ``establish_baseline`` does it for one directory: + + normalize the directory, build its graph, verify **every** invariant of the definitions against + it, map each result to a status (C-30), build the proofs, create version 0, and write, in one + unit of work, the definitions, one VERIFICATION record per result, the state and its lineage + head, and ``STATE_CREATED``. + +**Acceptance protocol (C-171, conservative; the specification leaves it to the experiment +protocol).** The baseline is created only if normalization is OK and yields resources, the +STRUCTURAL and CONSISTENCY gates PASS, no result is VERIFIER_ERROR, the request declares the fixture +eligible (Doc 13 §13), and the lineage has no baseline yet. Otherwise ``BaselineRefused`` is +raised, fail closed, and the only things stored are one ERROR record and one ERROR_OCCURRED event. +A VIOLATED or UNCERTAIN invariant is **accepted and recorded as it is**, never as PROTECTED: the +canonical scenario starts from public SSH. + +**The evidence carries the state's own hash (C-176).** ``to_evidence_submission`` stamps a result +with the hash of the resource set it verified; evidence about a *state* must carry that state's +``state_hash`` (Doc 11 §8, §44), and P6b's binding check compares exactly that value when a later +candidate carries an invariant forward from version 0 (``evaluate_proof`` condition P6). So the +baseline builds its submissions here, from the same result payload, once the state (and so its hash, +which does not depend on evidence ids) exists. + +**Idempotent.** Every id derives from ``baseline_id`` (``core.application.ids``), so a second call +is a duplicate: it rewrites nothing, returns the stored state with ``duplicate=True``, and raises +``ConflictingDuplicateError`` if the same ``baseline_id`` now describes another state. The only +caller of ``save_baseline`` in the repository (C-175). + +Time is the request's ``now``; the module reads no clock (R6 of the P6c prompt). +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from core.application import ids as oids +from core.application.recorder import StageRecorder +from core.dependency.build import build_graph +from core.dependency.invariants import SemanticBinding +from core.domain.audit import AuditEventType +from core.domain.enums import ( + InvariantStatus, + ProvenanceSourceKind, + VerificationResult, +) +from core.domain.errors import DomainValidationError +from core.domain.evidence import EvidenceKind, EvidenceSubmission +from core.domain.hashing import content_hash +from core.domain.ids import require_uuid +from core.domain.impact import ReasonCode, VerificationWorkItem, WorkPriority, WorkTargetType +from core.domain.invariant import Invariant, InvariantProof +from core.domain.jsonvalue import thaw_json, utc +from core.domain.lifecycle import RESULT_TO_STATUS +from core.domain.promotion import ConflictingDuplicateError +from core.domain.repositories import UnitOfWork +from core.domain.state import TrustedState +from core.domain.values import Provenance +from core.domain.verification import ( + ConstructionStatus, + VerificationReport, + VerificationResultRecord, +) +from core.promotion.commit import PromotionContext +from core.terraform_model.normalization import normalize_directory +from core.verification.dispatch import DEFAULT_VERIFIERS, VerifierTable +from core.verification.engine import ( + CONSISTENCY_ID, + EVENT_NAME, + STRUCTURAL_ID, + run_work_set, +) +from core.verification.inputs import VerificationInputs + +_GATES = (STRUCTURAL_ID, CONSISTENCY_ID) +_MAX_DIAGNOSTICS = 10 +_MAX_TEXT = 300 + + +class BaselineRefused(DomainValidationError): + """The baseline was not established (C-171). ``reasons`` lists every cause found; nothing was + stored but one ERROR record and one ERROR_OCCURRED event.""" + + def __init__(self, reasons: Sequence[str]) -> None: + self.reasons = tuple(reasons) + super().__init__("the baseline was refused: " + "; ".join(self.reasons)) + + +@dataclass(frozen=True) +class BaselineRequest: + """What to establish. ``eligible`` is the caller's declaration that the fixture may serve as a + baseline (Doc 13 §13); ``now`` is the one instant used for everything that is hashed.""" + + baseline_id: str + lineage_id: str + baseline_dir: Path + variables: Mapping[str, object] | None + definitions: tuple[Invariant, ...] + bindings: tuple[SemanticBinding, ...] + context: PromotionContext + now: datetime + eligible: bool + verifiers: VerifierTable = DEFAULT_VERIFIERS + registry_version: int = 1 + + def __post_init__(self) -> None: + require_uuid(self.baseline_id, "BaselineRequest.baseline_id") + require_uuid(self.lineage_id, "BaselineRequest.lineage_id") + if not isinstance(self.baseline_dir, Path): + raise DomainValidationError("BaselineRequest.baseline_dir must be a Path") + if not isinstance(self.eligible, bool): + raise DomainValidationError("BaselineRequest.eligible must be a bool") + if not isinstance(self.context, PromotionContext): + raise DomainValidationError("BaselineRequest.context must be a PromotionContext") + definitions = tuple(self.definitions) + if not definitions or not all(isinstance(d, Invariant) for d in definitions): + raise DomainValidationError("BaselineRequest.definitions must hold Invariant objects") + bindings = tuple(self.bindings) + if not all(isinstance(b, SemanticBinding) for b in bindings): + raise DomainValidationError("BaselineRequest.bindings must hold SemanticBinding") + object.__setattr__(self, "definitions", definitions) + object.__setattr__(self, "bindings", bindings) + object.__setattr__(self, "now", utc(self.now, "BaselineRequest.now")) + + +@dataclass(frozen=True) +class BaselineResult: + """Version 0, the verification report it rests on, the evidence id of each result (by + ``verification_id``), the status given to each invariant, and whether it already existed.""" + + state: TrustedState + report: VerificationReport + evidence_ids: Mapping[str, str] + statuses: Mapping[str, InvariantStatus] + duplicate: bool + + +class _Refuse(Exception): + def __init__(self, reasons: Sequence[str]) -> None: + self.reasons = tuple(reasons) + super().__init__("; ".join(self.reasons)) + + +def _clip(text: str) -> str: + return text if len(text) <= _MAX_TEXT else text[: _MAX_TEXT - 3] + "..." + + +def establish_baseline( + factory: Callable[[], UnitOfWork], request: BaselineRequest +) -> BaselineResult: + """Establish version 0 of ``request.lineage_id`` (see the module docstring). + + ``factory`` opens a unit of work. Raises ``BaselineRefused`` for a refusal, + ``ConflictingDuplicateError`` when the baseline id already names another state or record, and + lets an infrastructure failure propagate after the write has rolled back.""" + if not callable(factory): + raise DomainValidationError("establish_baseline: factory must open a unit of work") + if not isinstance(request, BaselineRequest): + raise DomainValidationError("establish_baseline: request must be a BaselineRequest") + recorder = StageRecorder(request.context) + try: + prepared = _prepare(request) + with factory() as uow: + return _write(uow, recorder, request, prepared) + except _Refuse as refusal: + _record_refusal(factory, recorder, request, refusal.reasons) + raise BaselineRefused(refusal.reasons) from None + + +# --- the verification of the baseline ---------------------------------------------------------- + + +@dataclass(frozen=True) +class _Prepared: + state_id: str + resources: Any + normalization_version: str + report: VerificationReport + statuses: Mapping[str, InvariantStatus] + evidence_ids: Mapping[str, str] + results: Mapping[str, VerificationResultRecord] + + +def _prepare(request: BaselineRequest) -> _Prepared: + """Everything that can be decided before anything is written; raises ``_Refuse``.""" + if not request.eligible: + raise _Refuse(["the fixture is not declared eligible as a baseline (Doc 13 §13)"]) + state_id = oids.baseline_state_id_for(request.baseline_id) + outcome = normalize_directory(request.baseline_dir, request.variables, now=request.now) + if outcome.status.value != "OK" or outcome.resource_set_hash is None: + shown = [_clip(f"{d.code}: {d.message}") for d in outcome.diagnostics[:_MAX_DIAGNOSTICS]] + raise _Refuse([f"normalization is not OK (status {outcome.status.value})", *shown]) + if not outcome.resources: + raise _Refuse(["the baseline has no resource to verify"]) + graph = build_graph( + outcome.resources, request.definitions, request.bindings, state_id=state_id, now=request.now + ) + inputs = VerificationInputs( + candidate_resources=outcome.resources, + parent_resources=None, + candidate_state_hash=outcome.resource_set_hash, + normalization_version=outcome.normalization_version, + candidate_id=state_id, + parent_state_id=None, + identity=None, + impact=None, + candidate_graph=graph, + trusted_graph=None, + invariants=request.definitions, + construction=ConstructionStatus.OK, + ) + scope = tuple(sorted(r.address for r in outcome.resources)) + work = [ + VerificationWorkItem( + target_id=f"INVARIANT:{d.invariant_id}:{d.version}", + target_type=WorkTargetType.INVARIANT, + reason=ReasonCode.CREATED_IN_SCOPE, # every resource of a baseline is new to it + scope=scope, + required_evidence="L1", + priority=WorkPriority.BLOCKING, + ) + for d in request.definitions + ] + instant = request.now + report = run_work_set( + work, + inputs, + clock=lambda: instant, + ids=oids.baseline_verification_ids(request.baseline_id), + verifiers=request.verifiers, + ) + results = {r.target_id: r for r in report.results} + reasons = _acceptance_problems(report, results) + if reasons: + raise _Refuse(reasons) + statuses = { + d.invariant_id: RESULT_TO_STATUS[results[f"INVARIANT:{d.invariant_id}:{d.version}"].result] + for d in request.definitions + } + evidence_ids = { + r.verification_id: oids.baseline_evidence_id_for( + request.baseline_id, "verification", r.target_id + ) + for r in report.results + } + return _Prepared( + state_id, + outcome.resources, + outcome.normalization_version, + report, + statuses, + evidence_ids, + results, + ) + + +def _acceptance_problems( + report: VerificationReport, results: Mapping[str, VerificationResultRecord] +) -> list[str]: + """Acceptance conditions (b) and (c) of C-171.""" + reasons = [] + for gate in _GATES: + record = results.get(gate) + if record is None or record.result is not VerificationResult.PASS: + shown = "missing" if record is None else record.result.value + reasons.append(f"the {gate} gate is {shown}, not PASS") + for record in report.results: + if record.result is VerificationResult.VERIFIER_ERROR: + reasons.append(f"{record.target_id}: VERIFIER_ERROR") + return reasons + + +# --- the write --------------------------------------------------------------------------------- + + +def _submission( + result: VerificationResultRecord, + evidence_id: str, + context: PromotionContext, + state: TrustedState, +) -> EvidenceSubmission: + """The VERIFICATION record of one result, bound to the baseline state and carrying its hash + (C-176). Same payload, kind, event name and provenance as ``to_evidence_submission``.""" + return EvidenceSubmission.create( + payload=thaw_json(result.evidence), + evidence_id=evidence_id, + run_id=context.run_id, + attempt_id=context.attempt_id, + correlation_id=context.correlation_id, + operation_id=context.operation_id, + kind=EvidenceKind.VERIFICATION, + event_name=EVENT_NAME, + provenance=Provenance( + source_kind=ProvenanceSourceKind.DERIVED, + source_component="core.verification", + source_reference=result.target_id, + observed_at=result.completed_at, + tool_name=result.verifier_id, + tool_version=result.verifier_version, + algorithm_version=result.verifier_version, + ), + created_at=result.completed_at, + state_id=state.state_id, + state_hash=state.state_hash, + ) + + +def _register(uow: UnitOfWork, definitions: Sequence[Invariant]) -> None: + for definition in definitions: + stored = uow.invariants.get_definition(definition.invariant_id, definition.version) + if stored is None: + uow.invariants.register_definition(definition) + elif stored.definition_hash() != definition.definition_hash(): + raise ConflictingDuplicateError( + f"{definition.invariant_id} v{definition.version} is already registered with " + "another definition (Doc 06 §4.2)" + ) + + +def _write( + uow: UnitOfWork, recorder: StageRecorder, request: BaselineRequest, prepared: _Prepared +) -> BaselineResult: + head = uow.trusted_states.get_current(request.lineage_id) + existing = uow.trusted_states.get(prepared.state_id) + if head is not None and head.state_id != prepared.state_id: + raise _Refuse([f"lineage {request.lineage_id} already has a baseline (SM-001)"]) + proofs = [ + InvariantProof.for_baseline( + invariant_id=d.invariant_id, + invariant_version=d.version, + status=prepared.statuses[d.invariant_id], + evidence_ids=( + prepared.evidence_ids[ + prepared.results[f"INVARIANT:{d.invariant_id}:{d.version}"].verification_id + ], + ), + verified_at=request.now, + ) + for d in request.definitions + ] + state = TrustedState.establish_baseline( + lineage_id=request.lineage_id, + resources=prepared.resources, + invariant_proofs=proofs, + evidence_refs=sorted({e for proof in proofs for e in proof.evidence_ids}), + normalization_version=prepared.normalization_version, + invariant_registry_version=request.registry_version, + now=request.now, + state_id=prepared.state_id, + ) + if existing is not None and ( + existing.state_hash != state.state_hash or existing.lineage_id != state.lineage_id + ): + raise ConflictingDuplicateError( + f"baseline {request.baseline_id} already names state {existing.state_id} with " + "another content (Doc 11 §38)" + ) + _register(uow, request.definitions) + for record in prepared.report.results: + recorder.append( + uow, + _submission( + record, prepared.evidence_ids[record.verification_id], request.context, state + ), + ) + if existing is None: + uow.trusted_states.save_baseline(state) + recorder.emit( + uow, + AuditEventType.STATE_CREATED, + event_id=oids.baseline_event_id_for(request.baseline_id, AuditEventType.STATE_CREATED), + payload={ + "lineage_id": state.lineage_id, + "version": 0, + "parent_state_id": None, + "state_hash": state.state_hash, + "baseline_id": request.baseline_id, + }, + when=request.now, + state_id=state.state_id, + ) + return BaselineResult( + state=existing or state, + report=prepared.report, + evidence_ids=prepared.evidence_ids, + statuses=prepared.statuses, + duplicate=existing is not None, + ) + + +def _record_refusal( + factory: Callable[[], UnitOfWork], + recorder: StageRecorder, + request: BaselineRequest, + reasons: Sequence[str], +) -> None: + """One ERROR record and one ERROR_OCCURRED event, keyed by the reasons so that the same refusal + again is a duplicate and another refusal is a record of its own.""" + key = content_hash({"reasons": list(reasons)})[:32] + with factory() as uow: + recorder.record_error( + uow, + stage="baseline", + error_type="BaselineRefused", + detail={"baseline_id": request.baseline_id, "reasons": list(reasons)}, + attempt_number=1, + at=request.now, + when=request.now, + evidence_id=oids.baseline_evidence_id_for(request.baseline_id, "error", key), + event_id=oids.baseline_event_id_for( + request.baseline_id, AuditEventType.ERROR_OCCURRED, key + ), + ) diff --git a/core/application/chain.py b/core/application/chain.py new file mode 100644 index 0000000..4bd67cc --- /dev/null +++ b/core/application/chain.py @@ -0,0 +1,544 @@ +"""The evidence chain of a candidate, and the checker that defines "complete" (P6c step 2; Doc 11 +§12, §43, §44, §51.3; Doc 06 §28; Doc 09 §26; C-172). + +``describe_chain`` reads back, in one open unit of work, everything the orchestrator wrote about one +candidate: its audit events in ``sequence`` order, its candidate-bound evidence in append order, its +decisions, the state a committed decision created and that state's binding record. It writes +nothing and never raises for a damaged record: what cannot be read becomes a problem. + +``EvidenceChain.problems()`` returns every violation of the rules of C-172, each as +``": "``: + + ORDER the events are not those of the candidate's status, in the order of C-172 + EVIDENCE-ORDER the candidate-bound records are not STATE, IDENTITY, DEPENDENCY, IMPACT and then + VERIFICATION records + NAMED-EVIDENCE an evidence id an event names does not exist, or a record no event names + INTEGRITY a record does not recompute its hash, or its payload is gone (Doc 11 §45) + BINDING a record is not bound to this candidate, its parent and its state hash + DECISION a decision row is incomplete, or missing for the candidate's status (§43) + ENDING the chain does not end as its status says: STATE_PROMOTED for a promotion, the + outcome event of the decision for a rejection, a retry or an escalation, + ERROR_OCCURRED for a FAILED candidate, and no STATE_PROMOTED otherwise + PROMOTED-STATE a promotion without its new state or its binding record bound to that state + STALE a STALE_PARENT decision that is not a rejection of a candidate whose parent + the lineage has left + UNREADABLE something could not be read + +``problems(finished=False)`` accepts a chain that stops early (a crash window): the rules that +describe the ending are not applied and the events and records may be a prefix of the full chain; +everything that was written must still be valid. + +The checker is production code so that P8's reports can call it. Standard library, ``core.domain``, +``core.promotion.ids`` and ``evidence`` only. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from core.domain.audit import AuditEvent, AuditEventType +from core.domain.enums import CandidateStatus, DecisionType +from core.domain.errors import DomainValidationError +from core.domain.evidence import ArtifactRef, EvidenceEvent, EvidenceKind, IntegrityStatus +from core.domain.promotion import DecisionReason, PromotionDecision +from core.domain.repositories import UnitOfWork +from core.domain.state import CandidateState, TrustedState +from core.promotion import ids as pids +from evidence.service import EvidenceService + +T = AuditEventType +K = EvidenceKind +S = CandidateStatus + +_VC = "VERIFICATION_COMPLETED" +_IGNORED = frozenset({T.PROMOTION_FAILED, T.EVIDENCE_VALIDITY_CHANGED}) +_HEAD = (T.CANDIDATE_CREATED, T.IDENTITY_COMPLETED, T.DEPENDENCY_COMPLETED, T.IMPACT_COMPLETED) +# After the verification events, by the candidate's status. PROMOTION_DECIDED comes from the +# decision, PROMOTION_REQUESTED from the orchestrator before a commit; a lost race stores a second +# decision (the stale REJECT, C-166); an escalation by revalidation follows a request (C-38). +_TAILS: Mapping[CandidateStatus, tuple[tuple[AuditEventType, ...], ...]] = { + S.ANALYZING: ((),), + S.PROMOTABLE: ((T.PROMOTION_DECIDED,), (T.PROMOTION_DECIDED, T.PROMOTION_REQUESTED)), + S.PROMOTED: ((T.PROMOTION_DECIDED, T.PROMOTION_REQUESTED, T.STATE_PROMOTED),), + S.REJECTED: ( + (T.PROMOTION_DECIDED, T.CANDIDATE_REJECTED), + (T.PROMOTION_DECIDED, T.PROMOTION_REQUESTED, T.PROMOTION_DECIDED, T.CANDIDATE_REJECTED), + ), + S.RETRY_REQUIRED: ((T.PROMOTION_DECIDED, T.RETRY_SCHEDULED),), + S.ESCALATED: ( + (T.PROMOTION_DECIDED, T.ESCALATED), + (T.PROMOTION_DECIDED, T.PROMOTION_REQUESTED, T.ESCALATED), + ), +} +_OUTCOME: Mapping[DecisionType, AuditEventType] = { + DecisionType.REJECT: T.CANDIDATE_REJECTED, + DecisionType.RETRY: T.RETRY_SCHEDULED, + DecisionType.ESCALATE: T.ESCALATED, +} +_STATUS_OF_OUTCOME: Mapping[CandidateStatus, DecisionType] = { + S.REJECTED: DecisionType.REJECT, + S.RETRY_REQUIRED: DecisionType.RETRY, + S.ESCALATED: DecisionType.ESCALATE, +} +_EVIDENCE_HEAD = (K.STATE, K.IDENTITY, K.DEPENDENCY, K.IMPACT) +_STAGE_EVENT = { + T.IDENTITY_COMPLETED: K.IDENTITY, + T.DEPENDENCY_COMPLETED: K.DEPENDENCY, + T.IMPACT_COMPLETED: K.IMPACT, +} + + +_BROKEN: Any = object() # a read that failed; the failure is in ``unreadable`` + + +@dataclass(frozen=True) +class ChainEvent: + """An audit event and the payload it points to.""" + + event: AuditEvent + payload: Mapping[str, Any] + + @property + def event_type(self) -> AuditEventType: + return self.event.event_type + + +@dataclass(frozen=True) +class EvidenceChain: + """Everything recorded about one candidate (see the module docstring).""" + + candidate_id: str + candidate: CandidateState | None + events: tuple[ChainEvent, ...] + evidence: tuple[EvidenceEvent, ...] + payloads: Mapping[str, Any] + named: Mapping[str, EvidenceEvent | None] + integrity: Mapping[str, str] + decisions: tuple[PromotionDecision, ...] + state: TrustedState | None + binding: EvidenceEvent | None + binding_integrity: str | None + head_state_id: str | None + unreadable: tuple[str, ...] + evidence_read: bool = True + + # --- what was recorded -------------------------------------------------------------------- + + def event_types(self) -> list[str]: + return [e.event_type.value for e in self.events] + + def evidence_kinds(self) -> list[str]: + return [e.kind.value for e in self.evidence] + + # --- the rules ---------------------------------------------------------------------------- + + def problems(self, *, finished: bool = True) -> list[str]: + found = [f"UNREADABLE: {what}" for what in self.unreadable] + if self.candidate is None: + return found or [f"UNREADABLE: candidate {self.candidate_id} is not stored"] + found += self._order(finished) + if self.evidence_read: # otherwise the failure above is the finding + found += self._evidence_order(finished) + found += self._named() + found += self._integrity() + found += self._binding() + found += self._decisions() + found += self._promoted_state() + found += self._stale(finished) + found += self._ending(finished) + return found + + def _types(self) -> list[AuditEventType]: + return [e.event_type for e in self.events] + + def _expected_sequences(self, candidate: CandidateState) -> list[list[str]]: + if candidate.status is S.FAILED: + return [[T.CANDIDATE_CREATED.value, T.ERROR_OCCURRED.value]] + sequences = [] + for tail in _TAILS.get(candidate.status, ()): + sequences.append([e.value for e in _HEAD] + [_VC + "+"] + [e.value for e in tail]) + return sequences + + @staticmethod + def _compress(types: list[str]) -> list[str]: + out: list[str] = [] + for item in types: + if item == _VC and out and out[-1] == _VC + "+": + continue + out.append(_VC + "+" if item == _VC else item) + return out + + def _order(self, finished: bool) -> list[str]: + candidate = self.candidate + assert candidate is not None + found: list[str] = [] + types = self._types() + decided = False + for kind in types: + if kind is T.PROMOTION_DECIDED: + decided = True + if kind is T.PROMOTION_FAILED and not decided: + found.append("ORDER: PROMOTION_FAILED comes before any decision") + # A stage that failed and was rerun leaves its ERROR_OCCURRED in the history (an error is + # not erased by a recovery); only a FAILED candidate's chain is built around it. + skipped = _IGNORED if candidate.status is S.FAILED else _IGNORED | {T.ERROR_OCCURRED} + kept = self._compress([t.value for t in types if t not in skipped]) + if candidate.status not in _TAILS and candidate.status is not S.FAILED: + if finished or kept[: len(_HEAD)] != [e.value for e in _HEAD][: len(kept)]: + found.append( + f"ORDER: no event order is defined for a {candidate.status.value} candidate" + ) + return found + allowed = self._expected_sequences(candidate) + if finished: + if kept not in allowed: + found.append( + f"ORDER: events are {kept}, expected one of {allowed} for a " + f"{candidate.status.value} candidate" + ) + return found + # an unfinished chain: any prefix of an ending + every = [*allowed, *self._all_prefix_targets()] + if not any(kept == seq[: len(kept)] for seq in every): + found.append(f"ORDER: events are {kept}, which is not a prefix of {every}") + return found + + def _all_prefix_targets(self) -> list[list[str]]: + full = [e.value for e in _HEAD] + [_VC + "+"] + return [full + [e.value for e in tail] for tails in _TAILS.values() for tail in tails] + + def _evidence_order(self, finished: bool) -> list[str]: + candidate = self.candidate + assert candidate is not None + kinds = self._compress_kinds( + [e.kind for e in self.evidence if e.kind is not K.ERROR or candidate.status is S.FAILED] + ) + if candidate.status is S.FAILED: + return [] if not kinds else [f"EVIDENCE-ORDER: a FAILED candidate is bound to {kinds}"] + expected = [k.value for k in _EVIDENCE_HEAD] + ["VERIFICATION+"] + if (finished and kinds != expected) or kinds != expected[: len(kinds)]: + return [f"EVIDENCE-ORDER: records are {kinds}, expected {expected}"] + return [] + + @staticmethod + def _compress_kinds(kinds: list[EvidenceKind]) -> list[str]: + out: list[str] = [] + for kind in kinds: + if kind is K.VERIFICATION and out and out[-1] == "VERIFICATION+": + continue + out.append("VERIFICATION+" if kind is K.VERIFICATION else kind.value) + return out + + def _named(self) -> list[str]: + found = [] + for evidence_id, record in self.named.items(): + if record is None: + found.append(f"NAMED-EVIDENCE: an event names evidence {evidence_id}, not stored") + by_event: dict[AuditEventType, set[str]] = {} + for item in self.events: + name = item.payload.get("evidence_id") + if isinstance(name, str): + by_event.setdefault(item.event_type, set()).add(name) + for event_type, kind in _STAGE_EVENT.items(): + for row in self.evidence: + if row.kind is kind and row.evidence_id not in by_event.get(event_type, set()): + found.append( + f"NAMED-EVIDENCE: {kind.value} record {row.evidence_id} is named by no " + f"{event_type.value} event" + ) + verification = by_event.get(T.VERIFICATION_COMPLETED, set()) + for row in self.evidence: + if row.kind is K.VERIFICATION and row.evidence_id not in verification: + found.append( + f"NAMED-EVIDENCE: VERIFICATION record {row.evidence_id} is named by no " + "VERIFICATION_COMPLETED event" + ) + return found + + def _integrity(self) -> list[str]: + return [ + f"INTEGRITY: evidence {evidence_id} is {status}" + for evidence_id, status in self.integrity.items() + if status != IntegrityStatus.VALID.value + ] + + def _binding(self) -> list[str]: + candidate = self.candidate + assert candidate is not None + found = [] + for row in self.evidence: + problems = [] + if row.candidate_id != candidate.candidate_id: + problems.append("another candidate") + if row.parent_state_id != candidate.parent_state_id: + problems.append("another parent state") + if candidate.state_hash is not None and row.state_hash != candidate.state_hash: + problems.append("another state hash") + if row.attempt_id is None: + problems.append("no attempt id") + if problems: + found.append(f"BINDING: {row.kind.value} record {row.evidence_id}: {problems}") + return found + + def _decisions(self) -> list[str]: + candidate = self.candidate + assert candidate is not None + found = [] + verification_ids = { + body["verification_id"] + for row in self.evidence + if row.kind is K.VERIFICATION + and isinstance(body := self.payloads.get(row.evidence_id), dict) + and "verification_id" in body + } + for d in self.decisions: + label = f"decision {d.decision_id}" + if d.candidate_id != candidate.candidate_id: + found.append(f"DECISION: {label} is about another candidate") + if d.parent_state_id != candidate.parent_state_id: + found.append(f"DECISION: {label} names another parent state") + if not d.policy_version: + found.append(f"DECISION: {label} records no policy version") + for stray in sorted(set(d.supporting_results) - verification_ids): + if not self.evidence_read: + break + found.append(f"DECISION: {label} is supported by {stray}, a result with no record") + for check in d.required_checks: + if check.verification_id is not None and ( + check.verification_id not in d.supporting_results + ): + found.append( + f"DECISION: {label}: required check {check.target_id} cites " + f"{check.verification_id}, which is not a supporting result" + ) + if d.reason is not DecisionReason.STALE_PARENT and not d.required_checks: + found.append(f"DECISION: {label} lists no required check") + if d.committed_state_id is not None and ( + self.state is None or self.state.state_id != d.committed_state_id + ): + found.append(f"DECISION: {label} is committed to a state that cannot be read") + found += self._status_decision(candidate) + return found + + def _status_decision(self, candidate: CandidateState) -> list[str]: + status = candidate.status + kinds = {d.decision for d in self.decisions} + committed = [d for d in self.decisions if d.committed_state_id is not None] + if status is S.PROMOTED: + ok = len(committed) == 1 and committed[0].decision is DecisionType.PROMOTE + return ( + [] if ok else ["DECISION: a PROMOTED candidate has no committed PROMOTE decision"] + ) + if committed: + return [f"DECISION: a {status.value} candidate has a committed decision"] + if status in _STATUS_OF_OUTCOME: + wanted = _STATUS_OF_OUTCOME[status] + # an escalation by revalidation keeps the PROMOTE it could not commit (C-38 case 2) + revalidated = status is S.ESCALATED and DecisionType.PROMOTE in kinds + if wanted not in kinds and not revalidated: + return [f"DECISION: a {status.value} candidate has no {wanted.value} decision"] + if status is S.PROMOTABLE and DecisionType.PROMOTE not in kinds: + return ["DECISION: a PROMOTABLE candidate has no PROMOTE decision"] + if status is S.FAILED and self.decisions: + return ["DECISION: a FAILED candidate has a decision (it has no state hash to bind)"] + return [] + + def _promoted_state(self) -> list[str]: + committed = [d for d in self.decisions if d.committed_state_id is not None] + if not committed: + return [] + decision = committed[0] + state, binding = self.state, self.binding + if state is None: + return [f"PROMOTED-STATE: decision {decision.decision_id} names a state not stored"] + found = [] + if state.commit_decision_id != decision.decision_id: + found.append("PROMOTED-STATE: the new state names another decision") + if binding is None: + return [*found, f"PROMOTED-STATE: no binding record for state {state.state_id}"] + if binding.kind is not K.STATE or binding.state_id != state.state_id: + found.append("PROMOTED-STATE: the binding record is not bound to the new state") + if binding.state_hash != state.state_hash: + found.append("PROMOTED-STATE: the binding record carries another state hash") + if binding.parent_state_id != decision.parent_state_id: + found.append("PROMOTED-STATE: the binding record names another parent") + if self.binding_integrity not in (None, IntegrityStatus.VALID.value): + found.append(f"PROMOTED-STATE: the binding record is {self.binding_integrity}") + return found + + def _stale(self, finished: bool) -> list[str]: + candidate = self.candidate + assert candidate is not None + found = [] + for d in self.decisions: + if d.reason is not DecisionReason.STALE_PARENT: + continue + if d.decision is not DecisionType.REJECT: + found.append(f"STALE: decision {d.decision_id} is STALE_PARENT but not a REJECT") + if candidate.status is not S.REJECTED: + found.append( + f"STALE: a STALE_PARENT decision, but the candidate is {candidate.status.value}" + ) + if self.head_state_id == candidate.parent_state_id: + found.append( + "STALE: a STALE_PARENT decision, but the candidate's parent is still the " + "lineage's current state" + ) + if finished: + last = self.events[-1] if self.events else None + if ( + last is None + or last.event_type is not T.CANDIDATE_REJECTED + or last.payload.get("reason") != DecisionReason.STALE_PARENT.value + ): + found.append("STALE: the chain does not end with REJECT STALE_PARENT") + return found + + def _ending(self, finished: bool) -> list[str]: + candidate = self.candidate + assert candidate is not None + found = [] + types = self._types() + # a recorded error (even one after the ending, when the call failed once the work was done) + # does not move the ending; only a FAILED candidate's chain is built around it + skipped = _IGNORED if candidate.status is S.FAILED else _IGNORED | {T.ERROR_OCCURRED} + types_for_ending = [t for t in types if t not in skipped] + if T.STATE_PROMOTED in types and candidate.status is not S.PROMOTED: + found.append(f"ENDING: STATE_PROMOTED on a {candidate.status.value} candidate") + if not finished: + return found + last = types_for_ending[-1] if types_for_ending else None + status = candidate.status + if status is S.PROMOTED: + if last is not T.STATE_PROMOTED: + found.append( + "ENDING: a promoted candidate's chain does not end with STATE_PROMOTED" + ) + elif status is S.FAILED: + if last is not T.ERROR_OCCURRED: + found.append("ENDING: a FAILED candidate's chain does not end with ERROR_OCCURRED") + elif status in _STATUS_OF_OUTCOME: + wanted = _OUTCOME[_STATUS_OF_OUTCOME[status]] + if last is not wanted: + found.append( + f"ENDING: a {status.value} candidate's chain ends with " + f"{None if last is None else last.value}, not {wanted.value}" + ) + else: + found.append(f"ENDING: the chain of a {status.value} candidate is not finished") + return found + + +# --- reading ----------------------------------------------------------------------------------- + + +def _try(unreadable: list[str], what: str, read: Any, default: Any = None) -> Any: + try: + return read() + except (DomainValidationError, KeyError, ValueError) as failure: + unreadable.append(f"{what}: {type(failure).__name__}: {failure}") + return default + + +def describe_chain(uow: UnitOfWork, candidate_id: str) -> EvidenceChain: + """Read the chain of ``candidate_id`` (read-only; see the module docstring).""" + unreadable: list[str] = [] + service = EvidenceService() + candidate = _try(unreadable, "candidate", lambda: uow.candidates.get(candidate_id)) + audit = _try(unreadable, "audit events", lambda: uow.audit.list_for_candidate(candidate_id), ()) + events = tuple( + ChainEvent( + e, + _try( + unreadable, + f"audit payload {e.event_id}", + lambda e=e: uow.evidence.resolve(ArtifactRef.parse(e.payload_ref)), + {}, + ), + ) + for e in audit + ) + listed = _try( + unreadable, "evidence", lambda: uow.evidence.list_for_candidate(candidate_id), _BROKEN + ) + evidence_read = listed is not _BROKEN + evidence: tuple[EvidenceEvent, ...] = () if listed is _BROKEN else tuple(listed) + payloads: dict[str, Any] = {} + integrity: dict[str, str] = {} + named: dict[str, EvidenceEvent | None] = {} + wanted = [row.evidence_id for row in evidence] + for item in events: + for key in ("evidence_id", "binding_evidence_id"): + value = item.payload.get(key) if isinstance(item.payload, dict) else None + if isinstance(value, str): + found = _try( + unreadable, f"evidence {value}", lambda v=value: uow.evidence.get(v), _BROKEN + ) + if found is not _BROKEN: + named[value] = found + wanted.append(value) + for evidence_id in dict.fromkeys(wanted): + if evidence_id in named and named[evidence_id] is None: + continue + status = _try( + unreadable, + f"integrity of {evidence_id}", + lambda v=evidence_id: service.verify_integrity(uow, v), + ) + if status is not None: + integrity[evidence_id] = status.value + record = named.get(evidence_id) or _try( + unreadable, f"evidence {evidence_id}", lambda v=evidence_id: uow.evidence.get(v) + ) + if record is not None and status is IntegrityStatus.VALID: + payloads[evidence_id] = _try( + unreadable, + f"payload of {evidence_id}", + lambda r=record: uow.evidence.resolve(ArtifactRef.parse(r.payload_ref)), + ) + decisions = tuple( + _try(unreadable, "decisions", lambda: uow.decisions.for_candidate(candidate_id), ()) + ) + committed = next((d for d in decisions if d.committed_state_id is not None), None) + state = binding = binding_integrity = None + if committed is not None and committed.committed_state_id is not None: + state = _try( + unreadable, + "the new state", + lambda: uow.trusted_states.get(committed.committed_state_id), + ) + binding_id = pids.evidence_id_for(committed.decision_id) + binding = _try(unreadable, "the binding record", lambda: uow.evidence.get(binding_id)) + if binding is not None: + status = _try( + unreadable, + "integrity of the binding record", + lambda: service.verify_integrity(uow, binding_id), + ) + binding_integrity = None if status is None else status.value + head = None + if candidate is not None: + current = _try( + unreadable, + "the lineage head", + lambda: uow.trusted_states.get_current(candidate.lineage_id), + ) + head = None if current is None else current.state_id + return EvidenceChain( + candidate_id=candidate_id, + candidate=candidate, + events=events, + evidence=evidence, + payloads=payloads, + named=named, + integrity=integrity, + decisions=decisions, + state=state, + binding=binding, + binding_integrity=binding_integrity, + head_state_id=head, + unreadable=tuple(unreadable), + evidence_read=evidence_read, + ) diff --git a/core/application/ids.py b/core/application/ids.py new file mode 100644 index 0000000..d8818cd --- /dev/null +++ b/core/application/ids.py @@ -0,0 +1,141 @@ +"""Identifiers the orchestrator and the baseline flow derive from a request (P6c; Doc 11 §38; +C-170 (4), C-171). + +Every id is ``uuid5(ORCHESTRATOR_NAMESPACE, ":")``, so the same request always +names the same decision, evidence records and audit events, and a rerun is a duplicate that the +database and the recorder recognise, never a conflict (Doc 11 §38). Nothing here draws a random +number or reads a clock. The namespace is a pinned constant: changing it, or a key, changes every +derived id (a test pins both). + +Verification ids are positional: the engine's id source is a zero-argument callable, so the n-th id +it asks for is ``uuid5(ns, "verification:::")``. The call sequence is fixed +by the sorted work set, hence by the candidate (C-170 (4)). +""" + +from __future__ import annotations + +import uuid +from collections.abc import Callable +from typing import Final + +from core.domain.audit import AuditEventType +from core.domain.errors import DomainValidationError +from core.domain.ids import require_uuid + +ORCHESTRATOR_NAMESPACE: Final = uuid.UUID("5e1d7a43-92c0-4b6f-8d3a-17f4c9e02b68") +STAGES: Final = ("state", "identity", "dependency", "impact", "verification", "error") + + +def _derived(key: str) -> str: + return str(uuid.uuid5(ORCHESTRATOR_NAMESPACE, key)) + + +def _attempt(value: object) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise DomainValidationError(f"attempt_number must be an integer >= 1, not {value!r}") + return value + + +def _stage(value: object) -> str: + if value not in STAGES: + raise DomainValidationError(f"stage must be one of {STAGES}, not {value!r}") + assert isinstance(value, str) + return value + + +def _part(value: object, name: str) -> str: + if not isinstance(value, str) or value == "": + raise DomainValidationError(f"{name} must be non-empty text, not {value!r}") + return value + + +def _suffix(extra: str | None, name: str) -> str: + return "" if extra is None else ":" + _part(extra, name) + + +def _event_type(value: object) -> str: + if not isinstance(value, AuditEventType): + raise DomainValidationError("event_type must be an AuditEventType") + return value.value + + +# --- an evaluation --------------------------------------------------------------------------- + + +def decision_id_for(candidate_id: str, attempt_number: int) -> str: + """The id of the decision about ``candidate_id`` at this attempt.""" + require_uuid(candidate_id, "candidate_id") + return _derived(f"decision:{candidate_id}:{_attempt(attempt_number)}") + + +def evidence_id_for( + candidate_id: str, attempt_number: int, stage: str, target_id: str | None = None +) -> str: + """The id of the evidence record of one stage (and one target, for a verification result).""" + require_uuid(candidate_id, "candidate_id") + return _derived( + f"evidence:{candidate_id}:{_attempt(attempt_number)}:{_stage(stage)}" + f"{_suffix(target_id, 'target_id')}" + ) + + +def event_id_for( + candidate_id: str, attempt_number: int, event_type: AuditEventType, key: str | None = None +) -> str: + """The id of one audit event of the evaluation (``key`` tells apart the events of one type).""" + require_uuid(candidate_id, "candidate_id") + return _derived( + f"event:{candidate_id}:{_attempt(attempt_number)}:{_event_type(event_type)}" + f"{_suffix(key, 'key')}" + ) + + +def verification_ids(candidate_id: str, attempt_number: int) -> Callable[[], str]: + """A fresh id source for ``run_work_set``: the n-th call returns the n-th derived id.""" + require_uuid(candidate_id, "candidate_id") + prefix = f"verification:{candidate_id}:{_attempt(attempt_number)}:" + counter = {"next": 0} + + def next_id() -> str: + value = _derived(prefix + str(counter["next"])) + counter["next"] += 1 + return value + + return next_id + + +# --- a baseline ------------------------------------------------------------------------------ + + +def baseline_state_id_for(baseline_id: str) -> str: + """The id of the version-0 trusted state that ``baseline_id`` establishes.""" + require_uuid(baseline_id, "baseline_id") + return _derived(f"baseline-state:{baseline_id}") + + +def baseline_evidence_id_for(baseline_id: str, stage: str, target_id: str | None = None) -> str: + require_uuid(baseline_id, "baseline_id") + return _derived( + f"baseline-evidence:{baseline_id}:{_stage(stage)}{_suffix(target_id, 'target_id')}" + ) + + +def baseline_event_id_for( + baseline_id: str, event_type: AuditEventType, key: str | None = None +) -> str: + require_uuid(baseline_id, "baseline_id") + return _derived(f"baseline-event:{baseline_id}:{_event_type(event_type)}{_suffix(key, 'key')}") + + +def baseline_verification_ids(baseline_id: str) -> Callable[[], str]: + """A fresh id source for the baseline verification run.""" + require_uuid(baseline_id, "baseline_id") + prefix = f"baseline-verification:{baseline_id}:" + counter = {"next": 0} + + def next_id() -> str: + value = _derived(prefix + str(counter["next"])) + counter["next"] += 1 + return value + + return next_id diff --git a/core/application/orchestrator.py b/core/application/orchestrator.py new file mode 100644 index 0000000..41d0c64 --- /dev/null +++ b/core/application/orchestrator.py @@ -0,0 +1,775 @@ +"""The evaluation orchestrator: one call from a stored baseline and a candidate configuration to a +decision and, for a PROMOTE, the next Trusted State (P6c step 4; Doc 03 §15; Doc 06 §20, §27; Doc 09 +§25, §26, §40; Doc 11 §12, §43; C-170, C-172, C-173, C-174, C-175). + + load parent -> build candidate -> normalize -> identity -> graph -> impact -> scope policy + -> verify -> decide -> record evidence -> record decision -> commit if PROMOTE + +**It coordinates only.** It owns no policy and no decision logic: every rule it uses lives in a +package that has passed its gate (the engines, ``decide``, ``record_decision``, +``commit_promotion``). It routes on the *type* of what a stage returned (a normalization that +produced no resource set, a decision kind, a stored row) and never on a verification result, an +invariant id or a policy value; a test reads this file with ``ast`` to keep it so. The emergency +switch needs no branch here: the policy built from ``promotion.automatic_enabled`` turns a +would-be PROMOTE into an ESCALATE (C-161). It imports neither ``sqlite3`` nor ``core.persistence`` +(it works through a factory that opens a unit of work, C-167), reads no clock and draws no +randomness (the clock is injected, every id is derived from the request, C-170 (4)). + +**Stages are separate units of work**, so a crash after stage k leaves records 1..k, all valid, and +a rerun with the same request completes: every record and event is idempotent by derived id, the +candidate resumes from its stored status, and a decision that is already stored is the decision. + +**The stages always run against the candidate's recorded parent** (``Patch.parent_state_id``), read +back from the database, and ``decide`` is given the **current** state: when the two differ the +candidate is stale and ``decide``'s own stale-first rule rejects it (Doc 09 §25) with real reports +behind the decision (C-173). The stale REJECT has the id ``rejection_decision_id_for(decision_id)``, +the one P6b gives the loser of a race. + +**Reruns.** A candidate that already has its decision (PROMOTED, REJECTED, RETRY_REQUIRED, +ESCALATED, FAILED) is not decided again: its stages are recomputed and re-recorded, so that another +directory under the same ids raises ``ConflictingDuplicateError``, and the stored decision comes +back with ``duplicate=True``. + +``checkpoint`` is a test-only hook, called with the name of a stage after its records committed. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime +from enum import StrEnum +from pathlib import Path +from typing import Any + +from core.application import ids as oids +from core.application.recorder import StageRecorder +from core.application.scope_policy import VerificationScopePolicy +from core.dependency.build import build_graph +from core.dependency.invariants import SemanticBinding +from core.domain.audit import AuditEventType +from core.domain.enums import CandidateSource, CandidateStatus, DecisionType +from core.domain.errors import DomainValidationError, StaleParentError +from core.domain.hashing import content_hash +from core.domain.ids import require_uuid +from core.domain.impact import ImpactReport +from core.domain.invariant import Invariant +from core.domain.patch import Patch +from core.domain.promotion import ( + ConflictingDuplicateError, + DecisionOutcome, + OracleVerdict, + PromotionDecision, + PromotionInputError, + PromotionPolicy, +) +from core.domain.repositories import UnitOfWork +from core.domain.state import ( + CandidateState, + TrustedState, + fail_building, + finish_building, + start_building, + transition_candidate, +) +from core.domain.verification import ConstructionStatus, VerificationReport +from core.identity.engine import compare +from core.impact.engine import compute_impact +from core.impact.inputs import PriorProof +from core.promotion import ids as pids +from core.promotion.commit import ( + CommitResult, + PromotionCommitError, + PromotionContext, + commit_promotion, + record_decision, +) +from core.promotion.decide import decide +from core.promotion.policy import dail_policy +from core.terraform_model.normalization import normalize_directory +from core.verification.dispatch import DEFAULT_VERIFIERS, VerifierTable +from core.verification.engine import run_work_set +from core.verification.inputs import VerificationInputs + +D = DecisionType +S = CandidateStatus +_FLAG = "promotion.automatic_enabled" +_TERMINAL = frozenset( + {S.PROMOTED, S.REJECTED, S.RETRY_REQUIRED, S.ESCALATED, S.FAILED} +) # the candidate already has its outcome + + +class EvaluationInputError(DomainValidationError): + """The request cannot be evaluated (malformed, or it names state the database does not hold).""" + + +class ResultKind(StrEnum): + """How an evaluation ended. DUPLICATE: the promotion had been committed already.""" + + PROMOTED = "PROMOTED" + REJECTED = "REJECTED" + RETRY_REQUIRED = "RETRY_REQUIRED" + ESCALATED = "ESCALATED" + STALE = "STALE" + CONSTRUCTION_FAILED = "CONSTRUCTION_FAILED" + DUPLICATE = "DUPLICATE" + + +_KIND_OF_DECISION: Mapping[DecisionType, ResultKind] = { + D.REJECT: ResultKind.REJECTED, + D.RETRY: ResultKind.RETRY_REQUIRED, + D.ESCALATE: ResultKind.ESCALATED, + D.PROMOTE: ResultKind.PROMOTED, +} + + +@dataclass(frozen=True) +class EvaluationRequest: + """One candidate to evaluate (C-170 (2)). ``patch`` is a record bound to the parent it was + written against; ``candidate_dir`` is the candidate configuration, and the patch text is not + applied to it (C-174). ``candidate_sequence`` is the candidate's place in its lineage.""" + + lineage_id: str + candidate_id: str + candidate_sequence: int + attempt_number: int + patch: Patch + candidate_dir: Path + variables: Mapping[str, object] | None + definitions: tuple[Invariant, ...] + bindings: tuple[SemanticBinding, ...] + context: PromotionContext + oracle_verdicts: tuple[OracleVerdict, ...] = () + + def __post_init__(self) -> None: + for name in ("lineage_id", "candidate_id"): + require_uuid(getattr(self, name), f"EvaluationRequest.{name}") + for name in ("candidate_sequence", "attempt_number"): + value = getattr(self, name) + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise EvaluationInputError(f"EvaluationRequest.{name} must be an integer >= 1") + if ( + not isinstance(self.patch, Patch) + or self.patch.source is not CandidateSource.FIXED_PATCH + ): + raise EvaluationInputError("EvaluationRequest.patch must be a FIXED_PATCH Patch") + if not isinstance(self.candidate_dir, Path): + raise EvaluationInputError("EvaluationRequest.candidate_dir must be a Path") + if not isinstance(self.context, PromotionContext): + raise EvaluationInputError("EvaluationRequest.context must be a PromotionContext") + definitions = tuple(self.definitions) + if not definitions or not all(isinstance(d, Invariant) for d in definitions): + raise EvaluationInputError("EvaluationRequest.definitions must hold Invariant objects") + bindings = tuple(self.bindings) + if not all(isinstance(b, SemanticBinding) for b in bindings): + raise EvaluationInputError("EvaluationRequest.bindings must hold SemanticBinding") + verdicts = tuple(self.oracle_verdicts) + if not all(isinstance(v, OracleVerdict) for v in verdicts): + raise EvaluationInputError("EvaluationRequest.oracle_verdicts must hold OracleVerdict") + object.__setattr__(self, "definitions", definitions) + object.__setattr__(self, "bindings", bindings) + object.__setattr__(self, "oracle_verdicts", verdicts) + + +@dataclass(frozen=True) +class EvaluationResult: + """What ``evaluate`` returns. ``decision`` is the stored decision (``None`` for a construction + failure), ``outcome`` the ``DecisionOutcome`` made in this call (``None`` on a rerun that had + its decision already), ``commit`` the result of ``commit_promotion`` when one ran, ``state`` the + new Trusted State of a promotion, ``candidate`` the candidate as stored afterwards. ``impact``, + ``verification`` and ``evidence_ids`` are the reports and the evidence-id map computed in this + call (reports are not persisted, C-167); ``duplicate`` is true when nothing new was decided.""" + + kind: ResultKind + candidate: CandidateState + decision: PromotionDecision | None = None + outcome: DecisionOutcome | None = None + commit: CommitResult | None = None + state: TrustedState | None = None + impact: ImpactReport | None = None + verification: VerificationReport | None = None + evidence_ids: Mapping[str, str] | None = None + duplicate: bool = False + + +class EvaluationOrchestrator: + """Evaluates candidates against a database (see the module docstring).""" + + def __init__( + self, + factory: Callable[[], UnitOfWork], + *, + config: Any, + scope_policy: VerificationScopePolicy, + clock: Callable[[], datetime], + policy_version: str, + verifiers: VerifierTable = DEFAULT_VERIFIERS, + policy_factory: Callable[..., PromotionPolicy] = dail_policy, + checkpoint: Callable[[str], None] | None = None, + ) -> None: + if not callable(factory): + raise EvaluationInputError("EvaluationOrchestrator: factory must open a unit of work") + if not isinstance(scope_policy, VerificationScopePolicy): + raise EvaluationInputError("EvaluationOrchestrator: scope_policy must have select()") + if not callable(clock): + raise EvaluationInputError("EvaluationOrchestrator: clock must be callable") + if not isinstance(policy_version, str) or not policy_version: + raise EvaluationInputError("EvaluationOrchestrator: policy_version must be text") + if checkpoint is not None and not callable(checkpoint): + raise EvaluationInputError("EvaluationOrchestrator: checkpoint must be callable") + try: + flag = config.get(_FLAG) + except (KeyError, AttributeError) as missing: + raise EvaluationInputError(f"the configuration has no {_FLAG}") from missing + if not isinstance(flag, bool): + raise EvaluationInputError(f"{_FLAG} must be a bool, not {flag!r}: refusing to start") + self._factory = factory + self._scope_policy = scope_policy + self._clock = clock + self._policy_version = policy_version + self._verifiers = verifiers + self._policy_factory = policy_factory + self._checkpoint = checkpoint + self._automatic = flag + + def evaluate(self, request: EvaluationRequest) -> EvaluationResult: + """Evaluate ``request`` (see the module docstring). Raises ``EvaluationInputError`` for a + request the database cannot satisfy, ``ConflictingDuplicateError`` when the ids are in use + with another content, ``PromotionCommitError`` as P6b defines it; any other exception + propagates after a best-effort ERROR_OCCURRED.""" + if not isinstance(request, EvaluationRequest): + raise EvaluationInputError("evaluate: request must be an EvaluationRequest") + run = _Run(self, request) + try: + return run.execute() + except ( + ConflictingDuplicateError, + PromotionInputError, + EvaluationInputError, + PromotionCommitError, + ): + raise + except Exception as failure: + run.record_failure(failure) + raise + + +class _Run: + """One call of ``evaluate``: the request, the candidate so far, and the stage reached.""" + + def __init__(self, orchestrator: EvaluationOrchestrator, request: EvaluationRequest) -> None: + self.o = orchestrator + self.r = request + self.recorder = StageRecorder(request.context) + self.stage = "load" + self.candidate: CandidateState | None = None + self.failed_now = False # this call is the one that moved the candidate to FAILED + + # --- small helpers ------------------------------------------------------------------------- + + def _passed(self, name: str) -> None: + if self.o._checkpoint is not None: + self.o._checkpoint(name) + + def _when(self) -> datetime: + return self.o._clock() + + @property + def _cid(self) -> str: + return self.r.candidate_id + + def _stored(self) -> CandidateState: + with self.o._factory() as uow: + stored = uow.candidates.get(self._cid) + if stored is None: + raise EvaluationInputError(f"candidate {self._cid} is not stored") + self.candidate = stored + return stored + + # --- the whole evaluation ------------------------------------------------------------------ + + def execute(self) -> EvaluationResult: + r, o = self.r, self.o + with o._factory() as uow: + current = uow.trusted_states.get_current(r.lineage_id) + parent = uow.trusted_states.get(r.patch.parent_state_id) + if current is None: + raise EvaluationInputError( + f"lineage {r.lineage_id} has no baseline to evaluate against" + ) + if parent is None or parent.lineage_id != r.lineage_id: + raise EvaluationInputError( + f"the patch was written against state {r.patch.parent_state_id}, which is not a " + f"state of lineage {r.lineage_id}" + ) + self.stage = "candidate" + candidate = self._create(parent) + at = candidate.created_at + self._passed("after_candidate") + self.stage = "build" + outcome = normalize_directory(r.candidate_dir, r.variables, now=at) + candidate = self._build(candidate, outcome) + self._passed("after_build") + if candidate.state_hash is None: + return self._construction_failed() + assert outcome.resource_set_hash is not None + self.stage = "identity" + identity = compare( + parent.resources, + outcome.resources, + list(outcome.moves), + parent_state_id=parent.state_id, + candidate_id=self._cid, + now=at, + ) + self._record( + lambda u: self.recorder.record_identity( + u, candidate, identity, attempt_number=r.attempt_number, at=at, when=self._when() + ) + ) + self._passed("after_identity") + self.stage = "dependency" + trusted_graph = build_graph( + parent.resources, r.definitions, r.bindings, state_id=parent.state_id, now=at + ) + candidate_graph = build_graph( + outcome.resources, r.definitions, r.bindings, state_id=self._cid, now=at + ) + self._record( + lambda u: self.recorder.record_dependency( + u, + candidate, + candidate_graph, + attempt_number=r.attempt_number, + at=at, + when=self._when(), + ) + ) + self._passed("after_dependency") + self.stage = "impact" + impact = compute_impact( + parent.resources, + outcome.resources, + identity, + trusted_graph, + candidate_graph, + r.definitions, + self._priors(parent), + parent_state_id=parent.state_id, + candidate_id=self._cid, + now=at, + ) + self._record( + lambda u: self.recorder.record_impact( + u, candidate, impact, attempt_number=r.attempt_number, at=at, when=self._when() + ) + ) + self._passed("after_impact") + self.stage = "verification" + work = o._scope_policy.select(impact, r.definitions) + inputs = VerificationInputs( + candidate_resources=outcome.resources, + parent_resources=parent.resources, + candidate_state_hash=outcome.resource_set_hash, + normalization_version=outcome.normalization_version, + candidate_id=self._cid, + parent_state_id=parent.state_id, + identity=identity, + impact=impact, + candidate_graph=candidate_graph, + trusted_graph=trusted_graph, + invariants=r.definitions, + construction=ConstructionStatus.OK, + ) + report = run_work_set( + work, + inputs, + clock=lambda: at, + ids=oids.verification_ids(self._cid, r.attempt_number), + verifiers=o._verifiers, + ) + evidence_ids = self._record( + lambda u: self.recorder.record_verification( + u, candidate, report, attempt_number=r.attempt_number, when=self._when() + ) + ) + self._passed("after_verification") + self.stage = "decision" + return self._decide(impact, report, evidence_ids, at) + + # --- the candidate ------------------------------------------------------------------------- + + def _record(self, write: Callable[[UnitOfWork], Any]) -> Any: + with self.o._factory() as uow: + return write(uow) + + def _create(self, parent: TrustedState) -> CandidateState: + """The stored candidate, or a new one: patch, CREATED then BUILDING, CANDIDATE_CREATED.""" + r = self.r + with self.o._factory() as uow: + stored = uow.candidates.get(self._cid) + patch = uow.patches.get(r.patch.patch_id) + if patch is not None and patch != r.patch: + raise ConflictingDuplicateError( + f"patch {r.patch.patch_id} is already stored with different content" + ) + if stored is not None: + if ( + stored.patch_id != r.patch.patch_id + or stored.patch_hash != r.patch.content_hash + or stored.parent_state_id != parent.state_id + or stored.candidate_sequence != r.candidate_sequence + or stored.lineage_id != r.lineage_id + ): + raise ConflictingDuplicateError( + f"candidate {self._cid} already exists for another patch, parent or " + "sequence" + ) + self.candidate = stored + return stored + if patch is None: + uow.patches.save(r.patch) + created = CandidateState.create( + parent=parent, + patch=r.patch, + candidate_sequence=r.candidate_sequence, + now=self._when(), + candidate_id=self._cid, + ) + building = start_building(created) + uow.candidates.create(created) + uow.candidates.save_transition(building) + self.recorder.emit( + uow, + AuditEventType.CANDIDATE_CREATED, + event_id=oids.event_id_for( + self._cid, r.attempt_number, AuditEventType.CANDIDATE_CREATED + ), + payload={ + "parent_state_id": parent.state_id, + "patch_id": r.patch.patch_id, + "source": r.patch.source.value, + "patch_hash": r.patch.content_hash, + "candidate_sequence": r.candidate_sequence, + "attempt_number": r.attempt_number, + }, + when=self._when(), + candidate_id=self._cid, + ) + self.candidate = building + return building + + def _build(self, candidate: CandidateState, outcome: Any) -> CandidateState: + """BUILDING to READY to ANALYZING, or to FAILED (Doc 06 §27); a rerun finds its own stored + status and checks that the directory is the one that was built.""" + r = self.r + usable = outcome.resource_set_hash is not None + if candidate.state_hash is not None or candidate.status is S.FAILED: + built = candidate.state_hash + if usable != (built is not None) or (usable and built != outcome.resource_set_hash): + raise ConflictingDuplicateError( + f"candidate {self._cid} was built from another configuration " + "(its state hash differs, Doc 11 §38)" + ) + if not usable: + if candidate.status is S.FAILED: + return candidate + return self._fail(candidate, outcome) + if candidate.status is S.BUILDING or candidate.status is S.CREATED: + if candidate.status is S.CREATED: + candidate = start_building(candidate) + try: + ready = finish_building(candidate, outcome.resources) + except DomainValidationError as refusal: + return self._fail(candidate, outcome, str(refusal)) + if ready.state_hash != outcome.resource_set_hash: + return self._fail( + candidate, outcome, "the candidate's state hash is not the normalizer's" + ) + analyzing = transition_candidate(ready, S.ANALYZING) + with self.o._factory() as uow: + uow.candidates.save_transition(ready) + self.recorder.record_candidate( + uow, ready, r.patch, attempt_number=r.attempt_number, at=ready.created_at + ) + uow.candidates.save_transition(analyzing) + self.candidate = analyzing + return analyzing + if candidate.status is S.READY: + analyzing = transition_candidate(candidate, S.ANALYZING) + with self.o._factory() as uow: + self.recorder.record_candidate( + uow, + candidate, + r.patch, + attempt_number=r.attempt_number, + at=candidate.created_at, + ) + uow.candidates.save_transition(analyzing) + self.candidate = analyzing + return analyzing + with self.o._factory() as uow: # a rerun: the same record again + self.recorder.record_candidate( + uow, candidate, r.patch, attempt_number=r.attempt_number, at=candidate.created_at + ) + return candidate + + def _fail( + self, candidate: CandidateState, outcome: Any, note: str | None = None + ) -> CandidateState: + """BUILDING to FAILED with the ERROR record and ERROR_OCCURRED (C-170 (3)).""" + r = self.r + shown = [f"{d.code}: {d.message}" for d in outcome.diagnostics[:10]] + reason = note or f"normalization is not OK (status {outcome.status.value})" + failed = fail_building(candidate, reason[:300]) + self.failed_now = True + with self.o._factory() as uow: + uow.candidates.save_transition(failed) + self.recorder.record_error( + uow, + stage="build", + error_type="ConstructionFailed", + detail={"reason": reason[:300], "diagnostics": [t[:300] for t in shown]}, + attempt_number=r.attempt_number, + at=candidate.created_at, + when=self._when(), + candidate=failed, + ) + self.candidate = failed + return failed + + def _construction_failed(self) -> EvaluationResult: + """The candidate is FAILED: no decision exists to return, only the candidate and, if it was + already FAILED before this call, the mark that nothing new was written.""" + return EvaluationResult( + kind=ResultKind.CONSTRUCTION_FAILED, + candidate=self._stored(), + duplicate=not self.failed_now, + ) + + def _priors(self, parent: TrustedState) -> list[PriorProof]: + by_id = {d.invariant_id: d for d in self.r.definitions} + priors = [] + for ref in parent.invariant_refs: + definition = by_id.get(ref.invariant_id) + if definition is None: + raise EvaluationInputError( + f"the parent state refers to {ref.invariant_id}, which the request does not " + "define" + ) + priors.append( + PriorProof( + ref.invariant_id, + ref.invariant_version, + ref.evidence_ids, + definition.verifier_id, + definition.verifier_version, + ) + ) + return priors + + # --- decide, record, commit ---------------------------------------------------------------- + + def _current(self) -> TrustedState: + with self.o._factory() as uow: + current = uow.trusted_states.get_current(self.r.lineage_id) + if current is None: + raise EvaluationInputError(f"lineage {self.r.lineage_id} has no current state") + return current + + def _decision( + self, + candidate: CandidateState, + impact: ImpactReport, + report: VerificationReport, + policy: PromotionPolicy, + at: datetime, + ) -> tuple[DecisionOutcome, bool]: + """``decide`` against the current state as it is now. When the candidate's parent is not + that state it is stale and ``decide`` says so (Doc 09 §25) under the derived id of the + stale rejection, the one P6b gives the loser of a race (C-166).""" + r = self.r + current = self._current() + stale = current.state_id != candidate.parent_state_id + base_id = oids.decision_id_for(self._cid, r.attempt_number) + outcome = decide( + candidate, + current, + impact, + report, + policy, + decision_id=pids.rejection_decision_id_for(base_id) if stale else base_id, + now=at, + attempt_number=r.attempt_number, + oracle_verdicts=r.oracle_verdicts, + ) + return outcome, stale + + def _decide( + self, + impact: ImpactReport, + report: VerificationReport, + evidence_ids: Mapping[str, str], + at: datetime, + ) -> EvaluationResult: + r, o = self.r, self.o + candidate = self._stored() + if candidate.status in _TERMINAL: + return self._stored_result(candidate, impact, report, evidence_ids) + policy = o._policy_factory( + policy_version=o._policy_version, + invariants=r.definitions, + automatic_enabled=o._automatic, + ) + outcome, stale = self._decision(candidate, impact, report, policy, at) + try: + with o._factory() as uow: + moved = record_decision(uow, outcome, candidate, r.context, now=at) + except StaleParentError: + # Another promotion landed between decide and record_decision: the head only moves + # forward, so deciding again can only give the stale rejection (Doc 06 §27). + outcome, stale = self._decision(candidate, impact, report, policy, at) + with o._factory() as uow: + moved = record_decision(uow, outcome, candidate, r.context, now=at) + self.candidate = moved + self._passed("after_decision") + kind = _KIND_OF_DECISION[outcome.decision.decision] + if outcome.decision.decision is not D.PROMOTE: + return EvaluationResult( + kind=ResultKind.STALE if stale else kind, + candidate=self._stored(), + decision=outcome.decision, + outcome=outcome, + impact=impact, + verification=report, + evidence_ids=evidence_ids, + ) + self.stage = "commit" + with o._factory() as uow: + self.recorder.emit( + uow, + AuditEventType.PROMOTION_REQUESTED, + event_id=oids.event_id_for( + self._cid, r.attempt_number, AuditEventType.PROMOTION_REQUESTED + ), + payload={ + "decision_id": outcome.decision.decision_id, + "parent_state_id": outcome.decision.parent_state_id, + }, + when=self._when(), + candidate_id=self._cid, + decision_id=outcome.decision.decision_id, + ) + self._passed("after_request") + committed = commit_promotion( + o._factory, + outcome, + moved, + impact, + report, + policy, + evidence_ids, + r.context, + now=at, + attempt_number=r.attempt_number, + oracle_verdicts=r.oracle_verdicts, + ) + return self._after_commit(outcome, committed, impact, report, evidence_ids) + + def _after_commit( + self, + outcome: DecisionOutcome, + committed: CommitResult, + impact: ImpactReport, + report: VerificationReport, + evidence_ids: Mapping[str, str], + ) -> EvaluationResult: + if committed.stale: + kind = ResultKind.STALE + elif committed.escalated: + kind = ResultKind.ESCALATED + elif committed.duplicate: + kind = ResultKind.DUPLICATE + else: + kind = ResultKind.PROMOTED + return EvaluationResult( + kind=kind, + candidate=self._stored(), + decision=committed.decision, + outcome=outcome, + commit=committed, + state=committed.state, + impact=impact, + verification=report, + evidence_ids=evidence_ids, + duplicate=committed.duplicate, + ) + + def _stored_result( + self, + candidate: CandidateState, + impact: ImpactReport, + report: VerificationReport, + evidence_ids: Mapping[str, str], + ) -> EvaluationResult: + """A rerun of a candidate that has its outcome: the stored decision, nothing written.""" + base_id = oids.decision_id_for(self._cid, self.r.attempt_number) + with self.o._factory() as uow: + decisions = uow.decisions.for_candidate(self._cid) + committed = next((d for d in decisions if d.committed_state_id is not None), None) + state = ( + None + if committed is None or committed.committed_state_id is None + else uow.trusted_states.get(committed.committed_state_id) + ) + decision = committed or (decisions[-1] if decisions else None) + if decision is None: + kind = ResultKind.CONSTRUCTION_FAILED + elif state is not None: + kind = ResultKind.DUPLICATE + elif decision.decision_id == pids.rejection_decision_id_for(base_id): + kind = ResultKind.STALE + else: + kind = _KIND_OF_DECISION[decision.decision] + return EvaluationResult( + kind=kind, + candidate=candidate, + decision=decision, + state=state, + impact=impact, + verification=report, + evidence_ids=evidence_ids, + duplicate=True, + ) + + # --- failures that are not decisions ------------------------------------------------------- + + def record_failure(self, failure: Exception) -> None: + """Best effort: an ERROR_OCCURRED event (and its record) for the stage that failed. If it + cannot be written nothing is claimed and the caller still gets the original exception.""" + candidate = self.candidate + if candidate is None: + return + key = content_hash( + {"stage": self.stage, "type": type(failure).__name__, "message": str(failure)[:200]} + )[:16] + try: + with self.o._factory() as uow: + self.recorder.record_error( + uow, + stage=self.stage, + error_type=type(failure).__name__, + detail={"message": str(failure)[:300]}, + attempt_number=self.r.attempt_number, + at=candidate.created_at, + when=self._when(), + candidate=candidate, + evidence_id=oids.evidence_id_for( + self._cid, self.r.attempt_number, "error", key + ), + event_id=oids.event_id_for( + self._cid, self.r.attempt_number, AuditEventType.ERROR_OCCURRED, key + ), + ) + except Exception: # best effort: the stage's own exception is what the caller gets + return diff --git a/core/application/recorder.py b/core/application/recorder.py new file mode 100644 index 0000000..3cde8ea --- /dev/null +++ b/core/application/recorder.py @@ -0,0 +1,477 @@ +"""The stage recorder: evidence records and audit events of one evaluation (P6c step 2; Doc 11 §5, +§8-§12, §25, §28, §38; Doc 06 §28; C-54, C-55, C-170 (4), C-172). + +``StageRecorder`` writes through ``evidence.service.EvidenceService`` and the audit port, always +in a unit of work the caller owns, so a stage's evidence and its event commit together. Every record +carries the run, attempt, correlation and operation ids of the ``PromotionContext`` (C-55); a record +about a candidate is bound to it (candidate id, parent state, the candidate's state hash). + +**Idempotent by id.** The ids are derived from the request (``core.application.ids``), so a rerun +writes the same records: the same content under an id is a duplicate and is accepted, another +content raises ``ConflictingDuplicateError``. Nothing is overwritten. A payload that holds a secret +is refused (``DomainValidationError``) and nothing is stored: the service would redact it, but a +secret in a stage payload is a defect to surface, not to hide. + +The engines draw a random ``report_id`` or ``graph_id`` on every run and offer no way to inject one; +both sit outside the report's content hash. The recorder leaves them out of the payload, so a rerun +that computes a report again records the same bytes (C-170 (4)); the report is identified by its +content hash, which the payload carries. + +Time: ``at`` is the deterministic instant of the evaluation (the candidate's ``created_at``); it is +the only time that enters a hash. ``when`` is the live clock and stamps the audit event, which the +duplicate test ignores (Doc 11 §38). +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from datetime import datetime +from typing import Any + +from core.application import ids as oids +from core.domain.audit import ( + AuditEventType, + ConflictingDuplicateEventError, +) +from core.domain.enums import ProvenanceSourceKind +from core.domain.errors import DomainValidationError +from core.domain.evidence import EvidenceConflictError, EvidenceKind, EvidenceSubmission +from core.domain.graph import DependencyGraph +from core.domain.identity import IdentityReport +from core.domain.impact import ImpactReport +from core.domain.jsonvalue import thaw_json, utc +from core.domain.patch import Patch +from core.domain.promotion import ConflictingDuplicateError +from core.domain.redaction import Redactor +from core.domain.repositories import UnitOfWork +from core.domain.state import CandidateState +from core.domain.values import Provenance +from core.domain.verification import VerificationReport +from core.promotion.commit import PromotionContext +from core.verification.engine import to_evidence_submission +from evidence.ids import CorrelationContext +from evidence.service import EvidenceService + +_COMPONENT = "core.application.orchestrator" +_VERSION = "1" + + +def _need(value: object, kind: type, name: str) -> None: + if not isinstance(value, kind): + raise DomainValidationError(f"StageRecorder: {name} must be a {kind.__name__}") + + +def _without(data: Mapping[str, Any], key: str) -> dict[str, Any]: + return {k: v for k, v in data.items() if k != key} + + +class StageRecorder: + """Records the stages of one evaluation (or of a baseline) for one ``PromotionContext``.""" + + def __init__( + self, + context: PromotionContext, + *, + service: EvidenceService | None = None, + redactor: Redactor | None = None, + ) -> None: + _need(context, PromotionContext, "context") + self._context = context + self._service = EvidenceService() if service is None else service + self._redactor = Redactor() if redactor is None else redactor + self._ctx = CorrelationContext( + context.run_id, context.correlation_id, context.operation_id, context.attempt_id + ) + + @property + def context(self) -> PromotionContext: + return self._context + + # --- the one place that writes ------------------------------------------------------------ + + def _check_secret(self, payload: Any, what: str) -> None: + if self._redactor.contains_secret(thaw_json(payload)): + raise DomainValidationError( + f"StageRecorder: the {what} payload contains a secret-like value; it is refused, " + "not stored (Doc 11 §28)" + ) + + def append(self, uow: UnitOfWork, submission: EvidenceSubmission) -> bool: + """Append a prepared evidence submission. True for a duplicate. A secret is refused; a + different content under the id raises ``ConflictingDuplicateError``.""" + _need(submission, EvidenceSubmission, "submission") + self._check_secret(json.loads(submission.payload_json), "evidence") + try: + return uow.evidence.append(submission).duplicate + except EvidenceConflictError as conflict: + raise ConflictingDuplicateError( + f"evidence {submission.event.evidence_id} was already stored with different " + f"content (Doc 11 §38): {conflict}" + ) from conflict + + def _record( + self, + uow: UnitOfWork, + *, + kind: EvidenceKind, + event_name: str, + payload: Mapping[str, Any], + evidence_id: str, + at: datetime, + source_reference: str, + candidate: CandidateState | None, + state_hash: str | None, + ) -> str: + self._check_secret(payload, kind.value) + moment = utc(at, "at") + try: + self._service.record( + uow, + kind=kind, + event_name=event_name, + payload=payload, + ctx=self._ctx, + provenance=Provenance( + source_kind=ProvenanceSourceKind.DERIVED, + source_component=_COMPONENT, + source_reference=source_reference, + observed_at=moment, + tool_name="evaluation-orchestrator", + tool_version=_VERSION, + algorithm_version=_VERSION, + ), + candidate_id=candidate.candidate_id if candidate is not None else None, + parent_state_id=None if candidate is None else candidate.parent_state_id, + state_hash=state_hash, + evidence_id=evidence_id, + now=moment, + ) + except EvidenceConflictError as conflict: + raise ConflictingDuplicateError( + f"evidence {evidence_id} was already stored with different content " + f"(Doc 11 §38): {conflict}" + ) from conflict + return evidence_id + + def emit( + self, + uow: UnitOfWork, + event_type: AuditEventType, + *, + event_id: str, + payload: Mapping[str, Any], + when: datetime, + state_id: str | None = None, + candidate_id: str | None = None, + decision_id: str | None = None, + ) -> None: + """Append one audit event (idempotent by ``event_id``).""" + if not isinstance(event_type, AuditEventType): + raise DomainValidationError("StageRecorder.emit: event_type must be an AuditEventType") + self._check_secret(payload, event_type.value) + try: + self._service.append_audit_event( + uow, + event_type=event_type, + ctx=self._ctx, + payload=dict(payload), + actor_id=self._context.actor_id, + state_id=state_id, + candidate_id=candidate_id, + decision_id=decision_id, + event_id=event_id, + now=utc(when, "when"), + ) + except ConflictingDuplicateEventError as conflict: + raise ConflictingDuplicateError( + f"audit event {event_id} was already stored with different content " + f"(Doc 11 §38): {conflict}" + ) from conflict + + # --- the stages --------------------------------------------------------------------------- + + def record_candidate( + self, + uow: UnitOfWork, + candidate: CandidateState, + patch: Patch, + *, + attempt_number: int, + at: datetime, + ) -> str: + """The STATE record of the built candidate: its hash, its patch and its parent.""" + _need(candidate, CandidateState, "candidate") + _need(patch, Patch, "patch") + if candidate.state_hash is None: + raise DomainValidationError("StageRecorder: the candidate has no state_hash yet") + if candidate.patch_id != patch.patch_id or candidate.patch_hash != patch.content_hash: + raise DomainValidationError("StageRecorder: the patch is not the candidate's patch") + return self._record( + uow, + kind=EvidenceKind.STATE, + event_name="candidate.built", + payload={ + "candidate_id": candidate.candidate_id, + "parent_state_id": candidate.parent_state_id, + "candidate_sequence": candidate.candidate_sequence, + "attempt_number": attempt_number, + "state_hash": candidate.state_hash, + "normalization_version": candidate.normalization_version, + "resource_count": len(candidate.resources), + "patch_id": patch.patch_id, + "patch_hash": patch.content_hash, + "patch_source": patch.source.value, + }, + evidence_id=oids.evidence_id_for(candidate.candidate_id, attempt_number, "state"), + at=at, + source_reference=candidate.candidate_id, + candidate=candidate, + state_hash=candidate.state_hash, + ) + + def _stage( + self, + uow: UnitOfWork, + *, + stage: str, + kind: EvidenceKind, + event_type: AuditEventType, + payload: Mapping[str, Any], + report_hash: str, + candidate: CandidateState, + attempt_number: int, + at: datetime, + when: datetime, + ) -> str: + evidence_id = self._record( + uow, + kind=kind, + event_name=f"{stage}.completed", + payload=payload, + evidence_id=oids.evidence_id_for(candidate.candidate_id, attempt_number, stage), + at=at, + source_reference=candidate.candidate_id, + candidate=candidate, + state_hash=candidate.state_hash, + ) + self.emit( + uow, + event_type, + event_id=oids.event_id_for(candidate.candidate_id, attempt_number, event_type), + payload={ + "evidence_id": evidence_id, + "report_hash": report_hash, + "parent_state_id": candidate.parent_state_id, + }, + when=when, + candidate_id=candidate.candidate_id, + ) + return evidence_id + + def record_identity( + self, + uow: UnitOfWork, + candidate: CandidateState, + report: IdentityReport, + *, + attempt_number: int, + at: datetime, + when: datetime, + ) -> str: + """The IDENTITY record and IDENTITY_COMPLETED.""" + _need(candidate, CandidateState, "candidate") + _need(report, IdentityReport, "report") + self._belongs(report.candidate_id, report.parent_state_id, candidate, "identity report") + return self._stage( + uow, + stage="identity", + kind=EvidenceKind.IDENTITY, + event_type=AuditEventType.IDENTITY_COMPLETED, + payload=_without(report.to_dict(), "report_id"), + report_hash=report.content_hash, + candidate=candidate, + attempt_number=attempt_number, + at=at, + when=when, + ) + + def record_dependency( + self, + uow: UnitOfWork, + candidate: CandidateState, + graph: DependencyGraph, + *, + attempt_number: int, + at: datetime, + when: datetime, + ) -> str: + """The DEPENDENCY record (the candidate's graph) and DEPENDENCY_COMPLETED.""" + _need(candidate, CandidateState, "candidate") + _need(graph, DependencyGraph, "graph") + if graph.state_id != candidate.candidate_id: + raise DomainValidationError("StageRecorder: the graph is of another state") + return self._stage( + uow, + stage="dependency", + kind=EvidenceKind.DEPENDENCY, + event_type=AuditEventType.DEPENDENCY_COMPLETED, + payload=_without(graph.to_dict(), "graph_id"), + report_hash=graph.graph_hash, + candidate=candidate, + attempt_number=attempt_number, + at=at, + when=when, + ) + + def record_impact( + self, + uow: UnitOfWork, + candidate: CandidateState, + report: ImpactReport, + *, + attempt_number: int, + at: datetime, + when: datetime, + ) -> str: + """The IMPACT record (candidate-bound, C-54) and IMPACT_COMPLETED.""" + _need(candidate, CandidateState, "candidate") + _need(report, ImpactReport, "report") + self._belongs(report.candidate_id, report.parent_state_id, candidate, "impact report") + return self._stage( + uow, + stage="impact", + kind=EvidenceKind.IMPACT, + event_type=AuditEventType.IMPACT_COMPLETED, + payload=_without(report.to_dict(), "report_id"), + report_hash=report.content_hash, + candidate=candidate, + attempt_number=attempt_number, + at=at, + when=when, + ) + + def record_verification( + self, + uow: UnitOfWork, + candidate: CandidateState, + report: VerificationReport, + *, + attempt_number: int, + when: datetime, + ) -> dict[str, str]: + """One VERIFICATION record and one VERIFICATION_COMPLETED per result, in report order. + Returns ``verification_id -> evidence_id`` (what ``commit_promotion`` is given).""" + _need(candidate, CandidateState, "candidate") + _need(report, VerificationReport, "report") + if report.candidate_id != candidate.candidate_id or ( + report.candidate_hash != candidate.state_hash + ): + raise DomainValidationError( + "StageRecorder: the verification report is of another candidate" + ) + found: dict[str, str] = {} + for result in report.results: + evidence_id = oids.evidence_id_for( + candidate.candidate_id, attempt_number, "verification", result.target_id + ) + self.append( + uow, + to_evidence_submission( + result, + evidence_id=evidence_id, + run_id=self._context.run_id, + attempt_id=self._context.attempt_id, + correlation_id=self._context.correlation_id, + operation_id=self._context.operation_id, + parent_state_id=candidate.parent_state_id, + ), + ) + found[result.verification_id] = evidence_id + self.emit( + uow, + AuditEventType.VERIFICATION_COMPLETED, + event_id=oids.event_id_for( + candidate.candidate_id, + attempt_number, + AuditEventType.VERIFICATION_COMPLETED, + result.target_id, + ), + payload={ + "evidence_id": evidence_id, + "verification_id": result.verification_id, + "target": result.target_id, + "result": result.result.value, + "verifier_id": result.verifier_id, + "verifier_version": result.verifier_version, + }, + when=when, + candidate_id=candidate.candidate_id, + ) + return found + + def record_error( + self, + uow: UnitOfWork, + *, + stage: str, + error_type: str, + detail: Mapping[str, Any], + attempt_number: int, + at: datetime, + when: datetime, + candidate: CandidateState | None = None, + evidence_id: str | None = None, + event_id: str | None = None, + ) -> str: + """An ERROR record and ERROR_OCCURRED. Bound to ``candidate`` when one is given; without + one (a refused baseline) it is unbound, and the caller names the ids.""" + payload = { + "stage": stage, + "error_type": error_type, + "attempt_number": attempt_number, + "detail": dict(detail), + } + # Doc 11 §44: evidence bound to a candidate carries its state hash. A candidate that failed + # to build has none, so its error record is unbound and names the candidate in the payload. + hash_of = None if candidate is None else candidate.state_hash + bound = candidate if hash_of is not None else None + if candidate is not None: + payload["candidate_id"] = candidate.candidate_id + evidence_id = evidence_id or oids.evidence_id_for( + candidate.candidate_id, attempt_number, "error" + ) + event_id = event_id or oids.event_id_for( + candidate.candidate_id, attempt_number, AuditEventType.ERROR_OCCURRED + ) + elif evidence_id is None or event_id is None: + raise DomainValidationError( + "StageRecorder.record_error: an error without a candidate needs its ids" + ) + stored = self._record( + uow, + kind=EvidenceKind.ERROR, + event_name="error.occurred", + payload=payload, + evidence_id=evidence_id, + at=at, + source_reference=stage, + candidate=bound, + state_hash=hash_of, + ) + self.emit( + uow, + AuditEventType.ERROR_OCCURRED, + event_id=event_id, + payload={"evidence_id": stored, "stage": stage, "error_type": error_type}, + when=when, + candidate_id=candidate.candidate_id if candidate is not None else None, + ) + return stored + + @staticmethod + def _belongs( + candidate_id: str, parent_state_id: str, candidate: CandidateState, what: str + ) -> None: + if candidate_id != candidate.candidate_id or parent_state_id != candidate.parent_state_id: + raise DomainValidationError(f"StageRecorder: the {what} is of another candidate") diff --git a/docs/BUILD_SEQUENCE.md b/docs/BUILD_SEQUENCE.md index de8ec3a..edb8d31 100644 --- a/docs/BUILD_SEQUENCE.md +++ b/docs/BUILD_SEQUENCE.md @@ -21,7 +21,8 @@ The build is split into 18 work packages (N-06 in `DECISIONS_REGISTER.md`). Ever | 11 | Oct 28–30 | P5a structural/security/baseline verifiers | | 12 | Oct 31–Nov 3 | P5b functional verifier → P5 gate, M2 | | 13 | Nov 4–5 | P6a promotion decision | -| 14 | Nov 6–9 | P6b promotion transaction → P6 gate, M3 | +| 14 | Nov 6–9 | P6b promotion commit (done) | +| 14b | Nov 6–9 | P6c orchestrator, baseline flow, fixed-patch loop → P6 gate, M3 | | 15 | Nov 10–13 | D3 real-world pool | | 16 | Nov 14–20 | D4 scenario generator | | 17 | Nov 21–28 | D5 oracle labels + calibration (free route, N-08) | diff --git a/docs/DECISIONS_REGISTER.md b/docs/DECISIONS_REGISTER.md index 11d5eff..b030bf4 100644 --- a/docs/DECISIONS_REGISTER.md +++ b/docs/DECISIONS_REGISTER.md @@ -181,3 +181,10 @@ This register records every conflict found between the 15 `CloudSpartanX_*` spec | C-167 | P6b: commit-time revalidation (Appendix B `validate_decision`) | `commit_promotion` receives the stored decision's `DecisionOutcome`, the candidate, the impact and verification reports, the policy and the evidence-id map; inside the transaction it (1) re-reads the current state and candidate from the database, (2) runs `decide` again with those and the same `decision_id`, `now` and `attempt_number`, and (3) requires the new decision's `content_hash` to equal the stored one and its `proof_plan` to be identical. Any difference is a revalidation failure (C-38 case 2). It also checks decision completeness (Doc 11 §43) against what the database holds (the decision row, the candidate, the evidence). Reports are not persisted in P6b (they arrive as arguments); that is a recorded limit for P8. Implementation choice (P6b): `commit_promotion` takes a factory that opens a unit of work, not one open unit, because the C-38 outcomes after a rollback (the REJECT decision of a stale loser, the ESCALATED candidate, the PROMOTION_FAILED event) are written in a separate transaction. **API as built (deviations from the signatures in the P6b prompt):** `commit_promotion(factory, outcome, candidate, impact, verification, policy, evidence_ids, context, *, now, attempt_number=None, oracle_verdicts=())` takes the factory (above), a `PromotionContext` (the run, correlation, operation and attempt ids that the evidence and audit records carry) and no `evaluations`: it builds them from the verification results and `evidence_ids` with `evaluate_from_results`. `record_decision(uow, outcome, candidate, context, *, now)` also takes the context and the clock. A revalidation failure also appends the ESCALATED event of Doc 11 §12 and stores the PROMOTE decision uncommitted if it was not stored. The comparison covers the proof plan as well as the decision hash, because the hash does not cover the evidence ids of a carried reference (a test pins it). | Proposed (P6b) | | C-168 | P6b: the "a promoted state needs its decision" trigger of C-165 against the existing persistence tests | C-165 asks for a trigger that refuses a trusted state with `version > 0` unless a PROMOTE decision row of its `commit_decision_id` for its parent exists. About 25 direct calls of `save_promoted` in the existing tests (`test_persistence_repositories.py`, `test_persistence_lifecycle.py`, `test_data_int_007.py`, `test_p2_gate.py` and `tests/persistence_builders.py`) promote states with arbitrary decision ids and no decision row, and would be refused. Section 10 R3 of the P6b prompt says that any change to an existing test other than the schema literal is a defect. The two statements cannot both hold. Options: (a) keep the trigger and make the existing tests insert a PROMOTE decision row before `save_promoted`, through the shared builder where possible; (b) keep DATA-INT-004 (the decision row points at the state, one way, once) and the unique indexes, and leave out the reverse trigger, enforcing "no promoted state without its decision" in `commit_promotion` and in the adapter's `save_promoted`; (c) keep the trigger but let it apply only to states promoted through the commit. **Decision (the user, 29 Oct 2026): (b).** DATA-INT-004 and the unique indexes stay in the database; "no promoted state without its PROMOTE decision" is enforced in `commit_promotion` (it writes the decision first and refuses a state without it) and tested there, not by a trigger. No existing test changes for this. Can be tightened with migration tooling before P8. | Decided 29 Oct 2026 (user); implemented in P6b | | C-169 | P6b: repeated failures of one commit leave one PROMOTION_FAILED event | The audit event ids of C-163 are derived, so that a redelivered event is a duplicate (Doc 11 §38). The PROMOTION_FAILED event of a failed commit (C-38 cases 2 and 3) is derived from the decision id and the failure kind, so a second failure of the same kind of the same decision adds no event: the first is kept and the later ones are only raised to the caller. Doc 11 §12 and §38 do not say whether every failed attempt is an event or whether an attempt number belongs in the id. Choice made here: one event per decision and kind, so that a retry storm cannot grow the log and nothing is lost for a retry that then succeeds. For the authors: if every attempt must be recorded, the id needs the attempt id (it is already on the event's context); the change is one line in `core/promotion/ids.py` and the tests that pin the id. | Open (P6b) | +| C-170 | P6c: the `EvaluationOrchestrator` | (1) **Contract.** `core/application/orchestrator.py`: `EvaluationOrchestrator(factory, *, config, scope_policy, clock, verifiers=DEFAULT_VERIFIERS, policy_version, policy_factory=dail_policy, checkpoint=None)` and one public method `evaluate(request) -> EvaluationResult`. It calls the stages in the order load parent, build candidate, normalize, identity, graph, impact, scope policy, verify, decide, record evidence, record decision, commit if PROMOTE, and nothing else. It holds no `if` on a verification result, an invariant id or a policy value: it routes only on the type of a stage outcome (a normalization status, a decision kind, a stored row). A test reads its source with `ast` and fails if it names `VerificationResult`, `InvariantStatus`, `PASS`, `FAIL` or `UNKNOWN`. It imports neither `sqlite3` nor `core.persistence`, reads no clock and draws no randomness (the contract test of P6c step 1). (2) **Request.** `EvaluationRequest`: `lineage_id`, `candidate_id`, `candidate_sequence`, `attempt_number` (>= 1), `patch` (a FIXED_PATCH `Patch`), `candidate_dir`, `variables`, `definitions`, `bindings`, `context` (a `PromotionContext`), `oracle_verdicts` (default none). **Addition to the prompt's list:** `candidate_sequence`, because `candidates` has `UNIQUE (lineage_id, candidate_sequence)` (schema 3) and the repository ports offer no way to allocate the next one; the caller owns it, as in the P6b builders. The patch text is not applied (C-174). (3) **Construction failure.** If normalization is not OK or the candidate cannot reach READY, the candidate goes BUILDING to FAILED (Doc 06 §27), an `ERROR` evidence record (unbound, with the candidate id in its payload: a FAILED candidate has no state hash and evidence bound to a candidate must carry one, Doc 11 §44) and an `ERROR_OCCURRED` event (bound to the candidate) are written, the trusted state is untouched, and `evaluate` returns `kind=CONSTRUCTION_FAILED`. No `PromotionDecision` is stored: a FAILED candidate has no `state_hash` (Doc 05 §8.1) and a decision binds one, so Doc 09 §20's "candidate construction failure: REJECT" is delivered as the FAILED candidate plus the error record. For the authors (R4 of the P6c prompt). (4) **Deterministic ids.** Every id the orchestrator mints is `uuid5(ORCHESTRATOR_NAMESPACE, ...)` (constant in `core/application/ids.py`): `decision_id` from `(candidate_id, attempt_number)`, stage evidence ids from `(candidate_id, attempt_number, stage[, target_id])`, audit event ids likewise. **Verification ids are positional:** the engine's id source is a zero-argument callable, so request, result and report ids are `uuid5` of `(candidate_id, attempt_number, call index)`; the call sequence is fixed by the sorted work set, hence by the candidate. Keying them by target would need a change to `core.verification`, which is outside this package. **Times:** everything that enters a hash (the candidate, the stage payloads, the verification clock, the decision) uses one instant, `candidate.created_at`, which is read from the injected clock once, when the candidate is first stored, and read back from the database on a rerun; only the audit event timestamps, which are excluded from the duplicate test (Doc 11 §38), use the live clock. Without this a rerun with a real clock would raise a conflict on the verification payload (`started_at`) and on the decision hash. The audit trail therefore records the live time of each event, while every evidence record carries the instant of the candidate's creation; a limit, listed in the report. The identity, dependency and impact engines draw a random `report_id` or `graph_id` per run (outside the content hash, no injection point); the recorder leaves those two ids out of the payload so that a rerun records the same bytes (a test pins it). (5) **Idempotent.** A second `evaluate` with an equal request, after any prefix of the first has committed, reaches the same end state: stage records and events are duplicates, the decision is the stored one, a committed promotion returns `duplicate=True`. The same ids with another candidate directory raise `ConflictingDuplicateError` (the recorder turns the evidence and audit conflict errors into it); nothing is overwritten. (6) **Routing.** After `decide`, the decision is stored with `record_decision` (always, Doc 09 §40) and only a PROMOTE continues to `commit_promotion`. The emergency switch needs no branch: the policy built from the configuration turns a would-be PROMOTE into ESCALATE (C-161). The orchestrator reads `promotion.automatic_enabled` from the `ResolvedConfig` it is given and refuses to start unless it is a `bool`. (7) **Failure outside decisions.** An exception in a stage that is not a decision (a verifier crash is a VERIFIER_ERROR result, not an exception) propagates after a best-effort `ERROR_OCCURRED` event; the candidate stays ANALYZING for the caller to retry. `PromotionCommitError` propagates as P6b defines it. Nothing is swallowed. | Proposed (P6c) | +| C-171 | P6c: baseline establishment protocol (Doc 06 §24, Doc 13 §13) | `core/application/baseline.py::establish_baseline(factory, request) -> BaselineResult`. Normalize the baseline directory, build its graph, verify **every** invariant of the definitions against it (the work items are the invariant items, `parent_resources = None`), map each result with `RESULT_TO_STATUS` (PASS to PROTECTED, FAIL to VIOLATED, UNKNOWN, UNSUPPORTED and VERIFIER_ERROR to UNCERTAIN, C-30), record one baseline VERIFICATION evidence record per result (`to_evidence_submission` with a `state_id` and no parent), build the proofs with `InvariantProof.for_baseline`, call `TrustedState.establish_baseline`, store the definitions and the evidence first (the evidence foreign key is deferred, C-59), then `save_baseline`, then append `STATE_CREATED`. **Acceptance protocol (conservative, for the authors; Doc 06 §24 leaves it to the experiment specification):** the baseline is created only if (a) normalization is OK, (b) the STRUCTURAL and CONSISTENCY gates PASS, (c) no result is VERIFIER_ERROR, and (d) the request declares the fixture eligible (`eligible=True`, the caller's word, Doc 13 §13). VIOLATED and UNCERTAIN statuses are recorded as they are, never as PROTECTED; **a baseline with a VIOLATED invariant is accepted**, because the canonical scenario starts from public SSH (INV-SEC-001 VIOLATED). Otherwise `BaselineRefused` is raised and nothing is stored except an `ERROR` record and an `ERROR_OCCURRED` event. The ids derive from the request's `baseline_id`; a second call is a duplicate. | Proposed (P6c) | +| C-172 | P6c: the evidence chain, and the checker that defines "complete" | Per `evaluate` the orchestrator records, all candidate-bound with the run, attempt and correlation ids of the context (C-55): a STATE record (the candidate's `state_hash`, the patch id and hash), an IDENTITY, a DEPENDENCY (the candidate graph) and an IMPACT record (the report `to_dict()`; IMPACT is candidate-bound, C-54), one VERIFICATION record per result (`to_evidence_submission`), and the audit events `CANDIDATE_CREATED, IDENTITY_COMPLETED, DEPENDENCY_COMPLETED, IMPACT_COMPLETED, VERIFICATION_COMPLETED` (one per result), then those of P6b (`PROMOTION_DECIDED` and the outcome event) with `PROMOTION_REQUESTED` between the decision and the commit of a PROMOTE, then `STATE_PROMOTED`. `core/application/chain.py::describe_chain(uow, candidate_id) -> EvidenceChain` reads them back in audit `sequence` order and `EvidenceChain.problems()` returns every violation of: the event order above; every evidence id an event names exists, resolves and recomputes its hash (integrity VALID); every candidate-bound record is bound to this candidate and its `state_hash`; the decision row is complete (Doc 11 §43); a PROMOTE chain ends with `STATE_PROMOTED` and a binding record bound to the new state; a non-PROMOTE chain ends with the outcome event of its decision and has no `STATE_PROMOTED`; a stale chain ends with REJECT `STALE_PARENT`; a FAILED candidate chain ends with `ERROR_OCCURRED`. The checker is production code so that P8 can call it, and each rule is tested to fire on a broken database. | Proposed (P6c) | +| C-173 | P6c: stage order and stale parents | `evaluate` reads the current state first. If the candidate's parent (`Patch.parent_state_id`) is not the current state, the candidate is stale (Doc 09 §25). **As the P6c prompt wrote it:** do not run the stages, build `decide`'s inputs with an empty work set and report. **Not implementable without inventing a report:** `decide` requires an `ImpactReport` and a `VerificationReport`; an `ImpactReport` carries the hashes of a trusted graph, a candidate graph and an identity report, which only the stages produce, and the prompt (R2) says to stop and ask rather than invent one. **Decided by the user during P6c (answer to the question asked at that point): run the stages against the candidate's recorded parent**, loaded from the database (`trusted_states.get(patch.parent_state_id)`, which is immutable and still stored), so that the reports are real; then call `decide` with the **current** state, whose stale-first rule (Doc 09 Appendix A) returns REJECT `STALE_PARENT` before it reads the reports. The stage records are written, so the stale chain is as complete as any other. The stale REJECT carries the derived id `core.promotion.ids.rejection_decision_id_for(decision_id)`, the id that P6b gives the loser of a race (C-166), so that a candidate has at most one of the PROMOTE and the REJECT under the same names. A stale candidate is still ANALYZING when decided (CREATED, BUILDING, READY, ANALYZING, REJECTED). **The current state is read again when the orchestrator decides, and a parent that moves between `decide` and `record_decision` is handled:** `record_decision` refuses a PROMOTE for a candidate whose parent is no longer current (`StaleParentError`, SM-010, from `candidate_after`); the orchestrator catches that one refusal, reads the current state once more and decides again, which can only give the stale rejection because the head only moves forward. Found by the P6c concurrency suite (9 of 28 schedules ended in the raw exception before the fix; a deterministic regression test pins the window). When the decision of the same candidate has been committed already, the parent is no longer current by construction and the stages run again against the recorded parent so that the duplicate check of C-170 (5) still sees them. | Proposed (P6c), decided by the user | +| C-174 | P6c: fixed patch only, patch text not applied | The patch is a record bound to the parent; `candidate_dir` is the candidate. The orchestrator stores the `Patch` and its hash and checks `patch.parent_state_id` against the candidate's parent; it does not check that the patch text produces the directory. A limit for P7b (patch validation) and P8: a request whose patch and directory disagree is evaluated as the directory says. | Proposed (P6c) | +| C-175 | P6c: sole callers of the repository writes (closes the C-168 (b) carry-forward) | R8 is extended to the repository writes: in `core/`, `evidence/`, `llm/` and `scripts/`, `TrustedStateRepository.save_promoted` is called only from `core.promotion.commit` (defined in `core.persistence`) and `save_baseline` only from `core.application.baseline` (defined in `core.persistence`). Tests are not scanned, as for R8. The check (`repository_write_violations` in `tests/contract/test_architecture_boundaries.py`) sees an attribute use and a string given to `getattr`, `hasattr` or `setattr`; its self-test fires for a forbidden caller and for each write named by the other write's permission. Nothing is loosened. | Proposed (P6c) | +| C-176 | P6c: baseline evidence carries the state's own hash (the prompt names `to_evidence_submission`) | C-171 says to record the baseline VERIFICATION evidence with `to_evidence_submission(..., parent_state_id=None, state_id=)`. That helper stamps the record with `result.candidate_hash`, the hash of the **resource set** that was verified. Evidence about a state must carry that state's `state_hash` (Doc 11 §8, §44), and P6b's binding check (`carried_problems` for a version-0 parent, `evaluate_proof` condition P6) compares exactly that value when a later candidate carries an unaffected invariant forward from version 0 (INV-SEC-003 in the canonical scenario). Probed during P6c step 3: with the helper's record `evaluate_proof` answers "P6: the evidence state_hash differs from the state", so every carry-forward from a baseline would be refused and every canonical PROMOTE would ESCALATE. **Implemented instead:** `core/application/baseline.py` builds the submission itself, from the same result payload, kind, event name and provenance as the helper, once the state exists; the state hash depends on the invariant statuses and resources but not on evidence ids, and the evidence ids are derived from `baseline_id`, so the order is possible. No change to `core.verification` or `core.domain`. For the authors: either this, or a `state_hash` argument on `to_evidence_submission` (a change to a package that has passed its gate). | Proposed (P6c) | diff --git a/docs/PHASE_GATES.md b/docs/PHASE_GATES.md index 3d4d1a9..f7496d8 100644 --- a/docs/PHASE_GATES.md +++ b/docs/PHASE_GATES.md @@ -184,7 +184,60 @@ budget in the core and violated-baseline INSUFFICIENT_EVIDENCE for the authors. Proposed; C-37 and C-41 Decided. C-45 enforced in carry_forward and in decide. ``` -## P6b–P10 — NOT STARTED +## P6b — Promotion commit + +```text +P6b mini-gate: PASSED (external review, 29 Oct 2026; PR #25 at b9b270a, merged before the +verdict, C-28 process lapse recorded an eighth time). Suite passes on Python 3.12 and 3.13 (3941 +tests); black, ruff and mypy clean. C-162 fixed (the P6a blocking condition). Reviewer probes: forged +evidence and a candidate that is not PROMOTABLE are refused and escalated, nothing committed; +15 of 18 reviewer mutations of commit.py and binding.py killed, the 3 survivors are redundant +checks. Limits: concurrency schedules and the fault table were not re-run by the reviewer beyond +the suite; most P6b tests were written after the code (author's disclosure). +Carry-forward: "no promoted state without its decision" is enforced in commit_promotion and +check_database, not by a trigger (C-168 b): P6c adds a test that only the commit module calls +save_promoted. C-169 (one PROMOTION_FAILED event per decision and kind) Open for the authors; +decision reports are not persisted (recorded limit for P8). C-162 to C-167 Proposed; C-38 and C-168 +Decided. +``` + +## P6 — Promotion (pending external review) + +```text +P6 gate (Doc 15 §13.3) and milestone M3 (Doc 15 §23): NOT DECIDED. Status written by the P6c author +as an input to the external review; no gate and no milestone is claimed. P6c mini-gate: NOT DECIDED. + +Evidence map (commands: py -3.12 -m unittest ; ids are in docs/DECISIONS_REGISTER.md): + safe candidate promotes ........ tests.unit.test_e2e_lifecycles (steps 7-9, third candidate to v2), + tests.unit.test_e2e_chains.TestSafePromotion + unsafe candidate rejects ....... tests.unit.test_e2e_lifecycles (steps 2-6), + tests.unit.test_e2e_chains.TestRejectedCandidate, + tests.unit.test_e2e_fixtures (18 fixtures vs the P6a pipeline) + stale candidate rejects ........ tests.unit.test_e2e_lifecycles (steps 10-11), + tests.unit.test_e2e_chains.TestStaleParent, + tests.unit.test_e2e_concurrency (28 schedules) + evidence chain complete ........ tests.unit.test_chain (one test per rule of C-172), + tests.unit.test_e2e_chains (six chains of Doc 11 §51.3), + tests.unit.test_e2e_properties + valid candidate advances once .. tests.unit.test_e2e_idempotency (reruns, a fault after every stage + and at every unit-of-work commit), tests.unit.test_e2e_concurrency, + P6b: tests.unit.test_promotion_commit*, test_promotion_concurrency + no unsafe candidate can advance the emergency switch (test_e2e_chains.TestEmergencySwitch), + test_e2e_properties (switch off: nothing promoted), the AST test + that the orchestrator holds no decision logic, C-175 sole callers + (tests.contract.test_architecture_boundaries) +Limits: impact and verification reports are not persisted (P8); fixed patches only, the patch text is +not applied to the directory (C-174, P7b); one SQLite file, the MVP's single promotion worker (Doc 09 +§39), nothing is claimed for several processes beyond the concurrency tests; the baseline acceptance +protocol is conservative and for the authors (C-171); hashed evidence carries the candidate's +creation instant, audit timestamps are live (C-170 (4)). +Carry-forward: C-169 Open (for the authors). C-170 to C-176 Proposed (C-173 decided by the user on +the stale route). C-176: baseline evidence is built in core.application.baseline because +to_evidence_submission stamps the resource-set hash, not the state's. Construction failure stores +no decision (C-170 (3), for the authors). 18 fixtures have a parent, the prompt said 17. +``` + +## P6c–P10 — NOT STARTED ## Temporary exceptions diff --git a/tests/contract/test_architecture_boundaries.py b/tests/contract/test_architecture_boundaries.py index 0d6f548..19438ce 100644 --- a/tests/contract/test_architecture_boundaries.py +++ b/tests/contract/test_architecture_boundaries.py @@ -264,7 +264,7 @@ def check_repository(roots: tuple[str, ...] | None = CHECKED_ROOTS) -> list[Viol # --- Source-level rules R8 to R10 (P1a step 14) --------------------------------------------------- -SOURCE_RULES = frozenset({"R8", "R9", "R10"}) +SOURCE_RULES = frozenset({"R8", "R9", "R10", "ORCH"}) # R8: only the state module and the Promotion Controller package may create a trusted state from a # candidate or mark a candidate promoted (SM-001, SM-002). R8_ALLOWED_MODULES = ("core.domain.state", "core.promotion") @@ -306,6 +306,103 @@ def sole_caller_violations(module: str, source: str) -> list[Violation]: ] +# P6c (C-175, closes the C-168 (b) carry-forward): the repository writes of a trusted state have one +# caller each. ``save_promoted`` is called only by the commit of the Promotion Controller, and +# ``save_baseline`` only by the baseline flow; ``core.persistence`` defines them. Tests are not +# scanned (as for R8). +R8_REPOSITORY_WRITE_CALLERS = { + "save_promoted": ("core.promotion.commit", "core.persistence"), + "save_baseline": ("core.application.baseline", "core.persistence"), +} + + +def repository_write_violations(module: str, source: str) -> list[Violation]: + """R8 for the repository writes: a use of ``save_promoted`` or ``save_baseline`` (as an + attribute, or as the name given to ``getattr``) outside the modules that may call it.""" + found: list[Violation] = [] + for node in ast.walk(ast.parse(source)): + name: str | None = None + if isinstance(node, ast.Attribute) and node.attr in R8_REPOSITORY_WRITE_CALLERS: + name = node.attr + elif ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in ("getattr", "hasattr", "setattr") + and len(node.args) >= 2 + and isinstance(node.args[1], ast.Constant) + and node.args[1].value in R8_REPOSITORY_WRITE_CALLERS + ): + name = str(node.args[1].value) + if name is not None and not any( + _within(module, allowed) for allowed in R8_REPOSITORY_WRITE_CALLERS[name] + ): + found.append(Violation("R8", module, name)) + return found + + +# P6c (C-170, R6 of the P6c prompt): the orchestrator coordinates through a factory and the +# injected clock and ids. It imports neither sqlite3 nor core.persistence, reads no clock, draws no +# randomness, and never calls ``decide`` after ``commit_promotion`` (a decision is made, then +# recorded, then committed, in that order). +ORCHESTRATOR_MODULE = "core.application.orchestrator" +ORCHESTRATOR_CLOCK_CALLS = frozenset( + { + ("datetime", "now"), + ("datetime", "utcnow"), + ("date", "today"), + ("time", "time"), + ("time", "monotonic"), + ("time", "perf_counter"), + ("uuid", "uuid1"), + ("uuid", "uuid4"), + } +) + + +def orchestrator_violations(source: str) -> list[Violation]: + """The coupling rules of the orchestrator on one source (nothing is imported or executed).""" + module = ORCHESTRATOR_MODULE + tree = ast.parse(source) + aliases = _import_aliases(tree) + found: list[Violation] = [] + first_commit: int | None = None + decides: list[int] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import | ast.ImportFrom): + names = ( + [alias.name for alias in node.names] + if isinstance(node, ast.Import) + else [node.module or ""] + ) + for name in names: + if _top(name) == "sqlite3" or _within(name, "core.persistence"): + found.append(Violation("ORCH", module, name)) + if isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name in ("uuid1", "uuid4", "random") or node.module == "random": + found.append(Violation("ORCH", module, alias.name)) + elif isinstance(node, ast.Call): + func = node.func + callee = func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + if callee == "commit_promotion": + first_commit = ( + node.lineno if first_commit is None else min(first_commit, node.lineno) + ) + if callee == "decide": + decides.append(node.lineno) + if isinstance(func, ast.Attribute): + receiver = _receiver(func.value, aliases) + if (receiver, func.attr) in ORCHESTRATOR_CLOCK_CALLS: + found.append(Violation("ORCH", module, f"{receiver}.{func.attr}")) + if receiver == "random": + found.append(Violation("ORCH", module, f"random.{func.attr}")) + if first_commit is not None: + found += [Violation("ORCH", module, "decide after commit_promotion")] * sum( + 1 for line in decides if line > first_commit + ) + return found + + def _import_aliases(tree: ast.AST) -> dict[str, str]: """Local name -> the name it was imported as, so ``T.promote`` and ``dt.now`` are seen through ``import ... as``/``from ... import ... as``.""" @@ -1108,6 +1205,117 @@ def test_the_sole_caller_check_can_fail(self) -> None: with self.subTest(module=module, allowed=True): self.assertEqual(sole_caller_violations(module, source), []) + def test_only_the_named_modules_call_the_repository_writes(self) -> None: + """C-175: ``save_promoted`` only from ``core.promotion.commit`` and ``save_baseline`` only + from ``core.application.baseline`` (``core.persistence`` defines them).""" + offenders = [] + for path in iter_python_files(CHECKED_ROOTS): + module, _ = module_name(path) + source = path.read_text(encoding="utf-8") + offenders.extend(str(v) for v in repository_write_violations(module, source)) + self.assertEqual(offenders, [], "\n".join(offenders)) + + def test_the_repository_write_check_can_fail(self) -> None: + promoted = "def f(uow, s):\n uow.trusted_states.save_promoted(s)\n" + baseline = "def f(uow, s):\n uow.trusted_states.save_baseline(s)\n" + reflected = 'def f(repo, s):\n getattr(repo, "save_promoted")(s)\n' + stored = "def f(repo):\n write = repo.save_baseline\n return write\n" + for module in ( + "core.application.orchestrator", + "core.application.recorder", + "core.promotion.newstate", + "core.promotion.decide", + "core.identity.engine", + "evidence.service", + "llm.adapter", + "scripts.dev", + ): + for source, name in ( + (promoted, "save_promoted"), + (baseline, "save_baseline"), + (reflected, "save_promoted"), + (stored, "save_baseline"), + ): + with self.subTest(module=module, name=name): + found = repository_write_violations(module, source) + self.assertEqual([(v.rule, v.imported) for v in found], [("R8", name)]) + # each write has its own caller: the other module's permission does not carry over + self.assertTrue(repository_write_violations("core.promotion.commit", baseline)) + self.assertTrue(repository_write_violations("core.application.baseline", promoted)) + + def test_the_repository_write_check_allows_the_named_callers(self) -> None: + promoted = "def f(uow, s):\n uow.trusted_states.save_promoted(s)\n" + baseline = "def f(uow, s):\n uow.trusted_states.save_baseline(s)\n" + for module in ("core.promotion.commit", "core.persistence.sqlite", "core.persistence"): + with self.subTest(module=module, name="save_promoted"): + self.assertEqual(repository_write_violations(module, promoted), []) + for module in ("core.application.baseline", "core.persistence.sqlite"): + with self.subTest(module=module, name="save_baseline"): + self.assertEqual(repository_write_violations(module, baseline), []) + # a definition is not a call, and an unrelated name is not the write + defined = ( + "class R:\n def save_promoted(self, s): ...\n def save_baseline(self, s): ...\n" + ) + self.assertEqual(repository_write_violations("core.domain.repositories", defined), []) + self.assertEqual( + repository_write_violations("core.application.x", "uow.patches.save(p)\n"), [] + ) + self.assertEqual( + repository_write_violations("core.application.x", "getattr(repo, 'save')(p)\n"), [] + ) + + def test_the_orchestrator_is_decoupled_when_it_exists(self) -> None: + """C-170, R6 of P6c: no sqlite3, no core.persistence, no clock, no randomness, and no + ``decide`` after ``commit_promotion``. (``test_orchestrator.py`` requires the module to + exist; this test checks the file when there is one.)""" + path = REPO_ROOT / "core" / "application" / "orchestrator.py" + if path.is_file(): + found = orchestrator_violations(path.read_text(encoding="utf-8")) + self.assertEqual(found, [], "\n".join(str(v) for v in found)) + + def test_the_orchestrator_exists_and_is_among_the_checked_modules(self) -> None: + path = REPO_ROOT / "core" / "application" / "orchestrator.py" + self.assertTrue(path.is_file()) + self.assertIn( + ORCHESTRATOR_MODULE, {module_name(p)[0] for p in iter_python_files(CHECKED_ROOTS)} + ) + + def test_the_orchestrator_check_can_fail(self) -> None: + late_decide = "decide after commit_promotion" + broken = { + "import sqlite3\n": "sqlite3", + "from sqlite3 import connect\n": "sqlite3", + "import core.persistence.sqlite\n": "core.persistence.sqlite", + "from core.persistence.sqlite import SqliteUnitOfWork\n": "core.persistence.sqlite", + "from core.persistence import sqlite\n": "core.persistence", + "from datetime import datetime\ndatetime.now()\n": "datetime.now", + "import datetime\ndatetime.datetime.now()\n": "datetime.now", + "from datetime import datetime as dt\ndt.utcnow()\n": "datetime.utcnow", + "import time\ntime.time()\n": "time.time", + "import time as t\nt.monotonic()\n": "time.monotonic", + "import random\nrandom.random()\n": "random.random", + "from random import choice\n": "choice", + "import uuid\nuuid.uuid4()\n": "uuid.uuid4", + "from uuid import uuid4\n": "uuid4", + "def f():\n commit_promotion(1)\n decide(2)\n": late_decide, + "def f(m):\n m.commit_promotion(1)\n m.decide(2)\n": late_decide, + } + for source, imported in broken.items(): + with self.subTest(source=source.splitlines()[-1]): + found = orchestrator_violations(source) + self.assertIn(("ORCH", imported), [(v.rule, v.imported) for v in found]) + + def test_the_orchestrator_check_allows_the_intended_coupling(self) -> None: + for source in ( + "def f():\n decide(1)\n commit_promotion(2)\n", + "import uuid\nuuid.uuid5(NS, 'x')\n", + "from datetime import datetime\ndatetime(2026, 1, 1)\n", + "from core.domain.state import CandidateState\nclock()\n", + "from core.persistence_like import x\n", + ): + with self.subTest(source=source.splitlines()[-1]): + self.assertEqual(orchestrator_violations(source), []) + def test_scripts_import_only_stdlib_and_first_party(self) -> None: """A14: no third-party imports in scripts/ (core/ and evidence/ are covered by R7).""" offenders = [] diff --git a/tests/promotion_e2e_builders.py b/tests/promotion_e2e_builders.py new file mode 100644 index 0000000..a9dc4b4 --- /dev/null +++ b/tests/promotion_e2e_builders.py @@ -0,0 +1,216 @@ +"""Builders for the P6c tests (chain checker, orchestrator, end-to-end suites). Not a test module. + +``ChainCase`` writes the evidence chain of one candidate by hand: it records the stages through the +production ``StageRecorder`` and then decides, records and commits with the P6a/P6b functions, in +the order of C-172. It is a *reference writer* for the checker tests: the checker +(``core.application.chain``) is shown to accept what it writes and to fire when a record is +missing, out of order or tampered. The orchestrator itself is tested in ``test_orchestrator.py`` +with ``describe_chain`` on the chains it writes. +""" + +from __future__ import annotations + +from typing import Any + +from core.application import ids as oids +from core.application.recorder import StageRecorder +from core.domain.audit import AuditEventType +from core.domain.enums import DecisionType +from core.domain.patch import Patch +from core.domain.promotion import DecisionOutcome +from core.domain.state import TrustedState +from core.promotion.commit import CommitResult, commit_promotion, record_decision +from core.promotion.decide import decide +from tests.domain_builders import patch_for, uid +from tests.identity_builders import NOW as PIPELINE_NOW +from tests.promotion_commit_builders import CONTEXT, NOW +from tests.promotion_pipeline import PipelineRun, fixtures_with_a_parent, pipeline +from tests.promotion_pipeline_commit import PipelineCommitCase + +SAFE = "canonical_safe_repair_promote_v1" +REGRESSION = "canonical_security_repair_breaks_function_reject_v1" +STAGES = ("state", "identity", "dependency", "impact", "verification") + + +def run_for( + fixture_id: str, *, candidate_id: str | None = None, sequence: int = 1, patch_n: int = 7 +) -> PipelineRun: + """The real P6a pipeline on one fixture (uncached: every call draws new report ids).""" + options: dict[str, Any] = {"sequence": sequence, "patch_n": patch_n} + if candidate_id is not None: + options["candidate_id"] = candidate_id + return pipeline(fixtures_with_a_parent()[fixture_id], **options) + + +class ChainCase(PipelineCommitCase): + """A database in which chains are written the way the orchestrator writes them.""" + + def setUp(self) -> None: + super().setUp() + self.recorder = StageRecorder(CONTEXT) + self.patch_numbers: dict[str, int] = {} + + def patch_of(self, run: PipelineRun) -> Patch: + return patch_for(run.trusted, n=self.patch_numbers.get(run.candidate.candidate_id, 7)) + + # --- the pieces ---------------------------------------------------------------------------- + + def current_state(self, run: PipelineRun) -> TrustedState: + with self.uow() as u: + state = u.trusted_states.get_current(run.trusted.lineage_id) + assert state is not None + return state + + def emit_created(self, run: PipelineRun, *, attempt: int = 1) -> None: + cid = run.candidate.candidate_id + patch = self.patch_of(run) + with self.uow() as u: + self.recorder.emit( + u, + AuditEventType.CANDIDATE_CREATED, + event_id=oids.event_id_for(cid, attempt, AuditEventType.CANDIDATE_CREATED), + payload={ + "parent_state_id": run.trusted.state_id, + "patch_id": patch.patch_id, + "source": patch.source.value, + }, + when=NOW, + candidate_id=cid, + ) + + def record_stage(self, run: PipelineRun, stage: str, *, attempt: int = 1) -> Any: + """One stage, in a unit of work of its own (a crash after it leaves it).""" + c, at = run.candidate, PIPELINE_NOW + with self.uow() as u: + if stage == "state": + return self.recorder.record_candidate( + u, c, self.patch_of(run), attempt_number=attempt, at=at + ) + if stage == "identity": + return self.recorder.record_identity( + u, c, run.inputs.identity, attempt_number=attempt, at=at, when=NOW + ) + if stage == "dependency": + return self.recorder.record_dependency( + u, c, run.inputs.candidate_graph, attempt_number=attempt, at=at, when=NOW + ) + if stage == "impact": + return self.recorder.record_impact( + u, c, run.impact, attempt_number=attempt, at=at, when=NOW + ) + return self.recorder.record_verification( + u, c, run.verification, attempt_number=attempt, when=NOW + ) + + def decide_and_record( + self, run: PipelineRun, *, attempt: int = 1, decision_id: str | None = None, **policy: Any + ) -> tuple[DecisionOutcome, Any]: + """``decide`` against the current state, then ``record_decision``.""" + cid = run.candidate.candidate_id + chosen = decision_id or oids.decision_id_for(cid, attempt) + policy_object = run.policy(**policy) + outcome = decide( + run.candidate, + self.current_state(run), + run.impact, + run.verification, + policy_object, + decision_id=chosen, + now=PIPELINE_NOW, + attempt_number=attempt, + ) + with self.uow() as u: + record_decision(u, outcome, run.candidate, CONTEXT, now=NOW) + return outcome, policy_object + + def request_and_commit( + self, + run: PipelineRun, + outcome: DecisionOutcome, + policy: Any, + evidence_ids: dict[str, str], + *, + attempt: int = 1, + ) -> CommitResult: + cid = run.candidate.candidate_id + decision = outcome.decision + with self.uow() as u: + self.recorder.emit( + u, + AuditEventType.PROMOTION_REQUESTED, + event_id=oids.event_id_for(cid, attempt, AuditEventType.PROMOTION_REQUESTED), + payload={ + "decision_id": decision.decision_id, + "parent_state_id": decision.parent_state_id, + }, + when=NOW, + candidate_id=cid, + decision_id=decision.decision_id, + ) + return commit_promotion( + lambda: self.uow(), + outcome, + run.candidate, + run.impact, + run.verification, + policy, + evidence_ids, + CONTEXT, + now=NOW, + attempt_number=attempt, + ) + + # --- whole chains -------------------------------------------------------------------------- + + def write_chain( + self, + run: PipelineRun, + *, + attempt: int = 1, + order: tuple[str, ...] = STAGES, + stages: tuple[str, ...] | None = None, + decide_now: bool = True, + **policy: Any, + ) -> tuple[DecisionOutcome | None, CommitResult | None]: + """CANDIDATE_CREATED, the stages (``order``, optionally only ``stages``), the decision, and + for a PROMOTE the request and the commit. The candidate must be stored (``self.store``).""" + self.emit_created(run, attempt=attempt) + evidence_ids: dict[str, str] = {} + for stage in order: + if stages is not None and stage not in stages: + continue + found = self.record_stage(run, stage, attempt=attempt) + if stage == "verification": + evidence_ids = found + if not decide_now: + return None, None + outcome, used = self.decide_and_record(run, attempt=attempt, **policy) + if outcome.decision.decision is not DecisionType.PROMOTE: + return outcome, None + return outcome, self.request_and_commit(run, outcome, used, evidence_ids, attempt=attempt) + + def promoted_chain(self, *, fixture_id: str = SAFE) -> PipelineRun: + """Install the world of ``fixture_id`` and promote its candidate with a complete chain.""" + run = run_for(fixture_id) + self.install(run) + self.store(run) + outcome, result = self.write_chain(run) + assert outcome is not None and result is not None and result.state is not None + return run + + def rejected_chain(self) -> PipelineRun: + run = run_for(REGRESSION) + self.install(run) + self.store(run) + outcome, result = self.write_chain(run) + assert outcome is not None and outcome.decision.decision is DecisionType.REJECT + assert result is None + return run + + def second_candidate(self, fixture_id: str = SAFE) -> PipelineRun: + """A candidate written against v0 beside the first (candidate id, sequence and patch of + its own); it is stale once the first is promoted.""" + run = run_for(fixture_id, candidate_id=uid(3998), sequence=2, patch_n=8) + self.patch_numbers[run.candidate.candidate_id] = 8 + self.store(run, sequence=2, patch_n=8) + return run diff --git a/tests/promotion_orchestrator_builders.py b/tests/promotion_orchestrator_builders.py new file mode 100644 index 0000000..38642fe --- /dev/null +++ b/tests/promotion_orchestrator_builders.py @@ -0,0 +1,168 @@ +"""Builders for the orchestrator tests (P6c steps 4 and 5). Not a test module. + +``OrchestratorCase`` gives a test a database, a factory over it, a configuration with the +emergency switch set either way, and the real ``EvaluationOrchestrator`` and baseline flow. Every +candidate of a test is a fixture directory written against a state of the database; nothing here +computes an expected decision. +""" + +from __future__ import annotations + +import dataclasses +import shutil +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from core.application.baseline import BaselineRequest, BaselineResult, establish_baseline +from core.application.config import ResolvedConfig, load_config +from core.application.orchestrator import ( + EvaluationOrchestrator, + EvaluationRequest, + EvaluationResult, +) +from core.application.scope_policy import DailScopePolicy +from core.dependency.invariants import SemanticBinding +from core.domain.enums import CandidateSource +from core.domain.patch import Patch +from core.domain.state import TrustedState +from core.persistence.schema import initialize_database +from core.promotion.commit import PromotionContext +from tests.domain_builders import LINEAGE, uid +from tests.fixture_catalog import REPO_ROOT +from tests.identity_builders import NOW as PIPELINE_NOW +from tests.impact_builders import registry +from tests.persistence_builders import RepoCase +from tests.pipeline_builders import fixed_clock + +FIXTURES = REPO_ROOT / "fixtures" +BASELINE_DIR = FIXTURES / "canonical_regression" / "baseline" +PATCH_SAFE_DIR = FIXTURES / "canonical_regression" / "patch_safe" +PATCH_REGRESSION_DIR = FIXTURES / "canonical_regression" / "patch_regression" +SOURCE, DESTINATION = "aws_instance.app", "aws_db_instance.database" +BASELINE_ID = uid(3100) + + +def config_with(automatic_enabled: bool) -> ResolvedConfig: + """The repository configuration, with ``promotion.automatic_enabled`` set in a (copied) reviewed + base file: the only way a safety-critical flag can be turned on (Doc 14 §37).""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "config" + shutil.copytree(REPO_ROOT / "config", root) + (root / "base" / "promotion.toml").write_text( + f"automatic_enabled = {'true' if automatic_enabled else 'false'}\n", encoding="utf-8" + ) + return load_config(root, "test", environ={}) + + +def definitions() -> tuple[Any, ...]: + return tuple(registry(SOURCE, DESTINATION).values()) + + +def bindings() -> tuple[SemanticBinding, ...]: + return (SemanticBinding("INV-FUNC-001", SOURCE, DESTINATION, "TCP", 5432),) + + +def context_for(n: int) -> PromotionContext: + """A context of its own (run, correlation, operation and attempt ids) for evaluation n.""" + return PromotionContext( + run_id=uid(0x7B00 + n), + correlation_id=uid(0x7B10 + n), + operation_id=uid(0x7B20 + n), + attempt_id=uid(0x7B30 + n), + actor_id=None, + ) + + +class OrchestratorCase(RepoCase): + """A database, a factory over it, and helpers that run the real orchestrator.""" + + automatic_enabled = True + + def setUp(self) -> None: + super().setUp() + self.opened = 0 + self.databases = 0 + + def fresh_database(self) -> None: + """Another empty database file for the rest of the test (the old one is left as it is).""" + self.databases += 1 + self.path = Path(self._tmp.name) / f"t{self.databases}.db" + initialize_database(self.path) + + def factory(self, checkpoint: Callable[[str], None] | None = None) -> Callable[[], Any]: + return lambda: self.uow(checkpoint=checkpoint) + + def orchestrator( + self, + *, + config: ResolvedConfig | None = None, + checkpoint: Callable[[str], None] | None = None, + db_checkpoint: Callable[[str], None] | None = None, + **options: Any, + ) -> EvaluationOrchestrator: + settings: dict[str, Any] = { + "config": config or config_with(self.automatic_enabled), + "scope_policy": DailScopePolicy(), + "clock": fixed_clock(60), + "policy_version": "policy-p6c", + "checkpoint": checkpoint, + } + settings.update(options) + return EvaluationOrchestrator(self.factory(db_checkpoint), **settings) + + def baseline( + self, directory: Path = BASELINE_DIR, *, baseline_id: str = BASELINE_ID, **changes: Any + ) -> BaselineResult: + request = BaselineRequest( + baseline_id=baseline_id, + lineage_id=LINEAGE, + baseline_dir=directory, + variables=None, + definitions=definitions(), + bindings=bindings(), + context=context_for(0), + now=PIPELINE_NOW, + eligible=True, + ) + return establish_baseline(self.factory(), dataclasses.replace(request, **changes)) + + def request( + self, + parent: TrustedState, + directory: Path, + *, + n: int = 1, + attempt: int = 1, + **changes: Any, + ) -> EvaluationRequest: + """The n-th candidate of the lineage, written against ``parent``.""" + base = EvaluationRequest( + lineage_id=parent.lineage_id, + candidate_id=uid(3400 + n), + candidate_sequence=n, + attempt_number=attempt, + patch=Patch.create( + source=CandidateSource.FIXED_PATCH, + content=f"fixed patch {n} for {directory.name}\n", + parent_state_id=parent.state_id, + now=PIPELINE_NOW, + patch_id=uid(3500 + n), + ), + candidate_dir=directory, + variables=None, + definitions=definitions(), + bindings=bindings(), + context=context_for(n), + ) + return dataclasses.replace(base, **changes) + + def current(self) -> TrustedState: + with self.uow() as u: + state = u.trusted_states.get_current(LINEAGE) + assert state is not None + return state + + def evaluate(self, request: EvaluationRequest, **options: Any) -> EvaluationResult: + return self.orchestrator(**options).evaluate(request) diff --git a/tests/unit/test_baseline.py b/tests/unit/test_baseline.py new file mode 100644 index 0000000..4d9ec78 --- /dev/null +++ b/tests/unit/test_baseline.py @@ -0,0 +1,433 @@ +"""The baseline establishment flow (P6c step 3; Doc 06 §24, Doc 13 §13; Doc 11 §8, §44; C-30, C-59, +C-171, C-176). + +HAND EXPECTATIONS (the fixture's own ``expected.json``, hand-written data, C-89, and the prompt): + +* the D2 ``canonical_regression/baseline`` directory gives version 0 with INV-SEC-001 VIOLATED + (public SSH is open) and INV-SEC-002, INV-SEC-003 and INV-FUNC-001 PROTECTED; +* the evidence of the four invariants is VERIFICATION evidence bound to version 0 and carrying its + state hash, so that it is proof for version 0 (``usable_as_proof``) and a later candidate can + carry an unaffected invariant forward from it; the evidence is written before the state, in the + same transaction, and ``STATE_CREATED`` is appended once; +* a second call with the same request is a duplicate and writes nothing; another configuration + under the same ``baseline_id`` raises ``ConflictingDuplicateError``; +* the baseline is refused (``BaselineRefused``), fail closed, when the fixture is not declared + eligible, normalization is not OK, a gate does not PASS, or a result is VERIFIER_ERROR; the only + things stored are one ERROR record and one ERROR_OCCURRED event, and a repeat stores nothing new; +* a failure at each checkpoint of the write leaves no state, no lineage head, no evidence. +""" + +from __future__ import annotations + +import dataclasses +import tempfile +import unittest +from pathlib import Path +from typing import Any +from unittest import mock + +from core.application import ids as oids +from core.application.baseline import ( + BaselineRefused, + BaselineRequest, + BaselineResult, + establish_baseline, +) +from core.dependency.invariants import SemanticBinding +from core.domain.audit import AuditEventType +from core.domain.enums import InvariantStatus, VerificationResult +from core.domain.errors import DomainValidationError, PersistenceError +from core.domain.evidence import EvidenceContext, EvidenceKind +from core.domain.promotion import ConflictingDuplicateError +from core.domain.verification import VerificationReason +from core.terraform_model.normalization import normalize_directory +from core.verification.dispatch import DEFAULT_VERIFIERS, REACHABILITY_CFG, VerifierTable +from core.verification.outcome import PredicateOutcome +from evidence.service import EvidenceService +from tests.domain_builders import LINEAGE, uid +from tests.identity_builders import NOW +from tests.impact_builders import registry +from tests.persistence_builders import RepoCase +from tests.promotion_commit_builders import CONTEXT, Gate, InjectedFault, dump +from tests.promotion_pipeline import ROOT, parent_statuses + +BASELINE_DIR = ROOT / "fixtures" / "canonical_regression" / "baseline" +SAFE_DIR = ROOT / "fixtures" / "canonical_regression" / "patch_safe" +SOURCE, DESTINATION = "aws_instance.app", "aws_db_instance.database" +BASELINE_ID = uid(3100) +SEC1, SEC2, SEC3, FUNC = "INV-SEC-001", "INV-SEC-002", "INV-SEC-003", "INV-FUNC-001" +# hand-derived from the fixture's expected.json: SEC-001 FAIL, the rest PASS +EXPECTED = { + SEC1: InvariantStatus.VIOLATED, + SEC2: InvariantStatus.PROTECTED, + SEC3: InvariantStatus.PROTECTED, + FUNC: InvariantStatus.PROTECTED, +} + + +def request_for(directory: Path = BASELINE_DIR, **changes: Any) -> BaselineRequest: + definitions = tuple(registry(SOURCE, DESTINATION).values()) + base = BaselineRequest( + baseline_id=BASELINE_ID, + lineage_id=LINEAGE, + baseline_dir=directory, + variables=None, + definitions=definitions, + bindings=(SemanticBinding(FUNC, SOURCE, DESTINATION, "TCP", 5432),), + context=CONTEXT, + now=NOW, + eligible=True, + ) + return dataclasses.replace(base, **changes) + + +def raising(_definition: Any, _inputs: Any) -> Any: + raise RuntimeError("the reachability verifier crashed") + + +def unknown(_definition: Any, _inputs: Any) -> Any: + return PredicateOutcome( + VerificationResult.UNKNOWN, VerificationReason.RULE_INDETERMINATE, None, {}, () + ) + + +def with_reachability(verifier: Any) -> VerifierTable: + return {**DEFAULT_VERIFIERS, REACHABILITY_CFG: verifier} + + +class BaselineCase(RepoCase): + def factory(self, checkpoint: Any = None) -> Any: + return lambda: self.uow(checkpoint=checkpoint) + + def establish(self, request: BaselineRequest | None = None, **options: Any) -> BaselineResult: + return establish_baseline( + self.factory(options.pop("checkpoint", None)), request or request_for() + ) + + def tables(self) -> dict[str, int]: + return { + t: self.count(t) + for t in ( + "trusted_states", + "lineage_heads", + "invariants", + "invariant_refs", + "evidence_events", + "audit_events", + ) + } + + +class TestTheCanonicalBaseline(BaselineCase): + def setUp(self) -> None: + super().setUp() + self.result = self.establish() + + def test_it_is_version_zero_of_the_requested_lineage(self) -> None: + state = self.result.state + self.assertEqual(state.version, 0) + self.assertIsNone(state.parent_state_id) + self.assertEqual(state.lineage_id, LINEAGE) + self.assertEqual(state.state_id, oids.baseline_state_id_for(BASELINE_ID)) + self.assertFalse(self.result.duplicate) + with self.uow() as u: + self.assertEqual(u.trusted_states.get_current(LINEAGE), state) + + def test_its_resources_are_those_of_the_directory(self) -> None: + outcome = normalize_directory(BASELINE_DIR, None, now=NOW) + self.assertEqual(self.result.state.resource_set_hash(), outcome.resource_set_hash) + + def test_the_statuses_are_those_of_the_fixture(self) -> None: + got = {r.invariant_id: r.status for r in self.result.state.invariant_refs} + self.assertEqual(got, EXPECTED) + self.assertEqual(got, parent_statuses("fixtures/canonical_regression/baseline")) + self.assertEqual(dict(self.result.statuses), EXPECTED) + self.assertEqual({r.invariant_version for r in self.result.state.invariant_refs}, {1}) + + def test_a_violated_invariant_is_recorded_as_violated_never_as_protected(self) -> None: + by = {r.invariant_id: r for r in self.result.state.invariant_refs} + self.assertIs(by[SEC1].status, InvariantStatus.VIOLATED) + + def test_the_definitions_are_registered(self) -> None: + with self.uow() as u: + for definition in registry(SOURCE, DESTINATION).values(): + self.assertEqual( + u.invariants.get_definition(definition.invariant_id, 1), definition + ) + + def test_there_is_one_state_bound_verification_record_per_result(self) -> None: + results = self.result.report.results + with self.uow() as u: + rows = [u.evidence.get(self.result.evidence_ids[r.verification_id]) for r in results] + self.assertEqual(len(rows), len(results)) + self.assertEqual(len(results), 6) # four invariants and the two gates + for row, result in zip(rows, results, strict=True): + assert row is not None + with self.subTest(target=result.target_id): + self.assertIs(row.kind, EvidenceKind.VERIFICATION) + self.assertEqual(row.state_id, self.result.state.state_id) + self.assertIsNone(row.candidate_id) + self.assertIsNone(row.parent_state_id) + self.assertEqual(row.state_hash, self.result.state.state_hash) + self.assertEqual(row.run_id, CONTEXT.run_id) + self.assertEqual(row.correlation_id, CONTEXT.correlation_id) + self.assertEqual( + row.evidence_id, + oids.baseline_evidence_id_for(BASELINE_ID, "verification", result.target_id), + ) + + def test_each_invariant_reference_carries_its_own_evidence(self) -> None: + by_target = {r.target_id: r for r in self.result.report.results} + for ref in self.result.state.invariant_refs: + record = by_target[f"INVARIANT:{ref.invariant_id}:{ref.invariant_version}"] + self.assertEqual(ref.evidence_ids, (self.result.evidence_ids[record.verification_id],)) + self.assertEqual( + list(self.result.state.evidence_refs), + sorted({e for ref in self.result.state.invariant_refs for e in ref.evidence_ids}), + ) + + def test_the_evidence_is_proof_for_version_zero(self) -> None: + """What carry-forward needs (C-164): P6 compares the record's hash with the state's.""" + service = EvidenceService() + context = EvidenceContext.state(self.result.state.state_id) + with self.uow() as u: + for ref in self.result.state.invariant_refs: + for evidence_id in ref.evidence_ids: + check = service.usable_as_proof(u, evidence_id, context) + self.assertTrue(check.usable, (ref.invariant_id, check.reasons)) + + def test_state_created_is_appended_once(self) -> None: + with self.uow() as u: + events = [ + e + for e in u.audit.list_for_correlation(CONTEXT.correlation_id) + if e.event_type is AuditEventType.STATE_CREATED + ] + [event] = events + self.assertEqual(event.state_id, self.result.state.state_id) + self.assertEqual( + event.event_id, + oids.baseline_event_id_for(BASELINE_ID, AuditEventType.STATE_CREATED), + ) + + def test_the_gates_passed(self) -> None: + by = {r.target_id: r.result for r in self.result.report.results} + self.assertIs(by["STRUCTURAL"], VerificationResult.PASS) + self.assertIs(by["CONSISTENCY"], VerificationResult.PASS) + + +class TestTheWriteOrder(BaselineCase): + def test_the_evidence_is_written_before_the_state_in_one_unit_of_work(self) -> None: + log: list[str] = [] + opened: list[int] = [] + + class Repo: + def __init__(self, inner: Any, name: str) -> None: + self._inner, self._name = inner, name + + def __getattr__(self, method: str) -> Any: + target = getattr(self._inner, method) + if not callable(target): + return target + + def call(*args: Any, **kwargs: Any) -> Any: + log.append(f"{self._name}.{method}") + return target(*args, **kwargs) + + return call + + class Spy: + def __init__(self, inner: Any) -> None: + self._inner = inner + + def __enter__(self) -> Spy: + self._inner.__enter__() + opened.append(1) + return self + + def __exit__(self, *exc: Any) -> Any: + return self._inner.__exit__(*exc) + + def __getattr__(self, name: str) -> Any: + return Repo(getattr(self._inner, name), name) + + establish_baseline(lambda: Spy(self.uow()), request_for()) + evidence = [i for i, entry in enumerate(log) if entry == "evidence.append"] + state = log.index("trusted_states.save_baseline") + self.assertTrue(evidence) + self.assertTrue(all(i < state for i in evidence), log) + self.assertEqual(len(opened), 1) # one unit of work, so one transaction + + +class TestIdempotency(BaselineCase): + def test_a_second_call_is_a_duplicate_and_writes_nothing(self) -> None: + first = self.establish() + before = dump(self.path) + second = self.establish() + self.assertTrue(second.duplicate) + self.assertEqual(second.state, first.state) + self.assertEqual(dict(second.evidence_ids), dict(first.evidence_ids)) + self.assertEqual(dump(self.path), before) + + def test_another_configuration_under_the_same_baseline_id_conflicts(self) -> None: + self.establish() + before = dump(self.path) + with self.assertRaises(ConflictingDuplicateError): + self.establish(request_for(SAFE_DIR)) + self.assertEqual(dump(self.path), before) + + def test_a_second_baseline_for_the_same_lineage_is_refused(self) -> None: + self.establish() + with self.assertRaises(BaselineRefused) as refused: + self.establish(request_for(baseline_id=uid(3101))) + self.assertTrue(any("already" in r for r in refused.exception.reasons)) + self.assertEqual(self.count("trusted_states"), 1) + + +class TestRefusals(BaselineCase): + """Each refusal stores one ERROR record and one ERROR_OCCURRED event and nothing else.""" + + def assert_refused(self, request: BaselineRequest, needle: str, **options: Any) -> None: + with self.assertRaises(BaselineRefused) as refused: + self.establish(request, **options) + self.assertTrue( + any(needle in reason for reason in refused.exception.reasons), + (needle, refused.exception.reasons), + ) + self.assertEqual( + self.tables(), + { + "trusted_states": 0, + "lineage_heads": 0, + "invariants": 0, + "invariant_refs": 0, + "evidence_events": 1, + "audit_events": 1, + }, + ) + with self.uow() as u: + [error] = u.evidence.list_for_correlation(CONTEXT.correlation_id) + [event] = u.audit.list_for_correlation(CONTEXT.correlation_id) + self.assertIs(error.kind, EvidenceKind.ERROR) + self.assertIsNone(error.state_id) + self.assertIs(event.event_type, AuditEventType.ERROR_OCCURRED) + + def test_a_fixture_not_declared_eligible(self) -> None: + self.assert_refused(request_for(eligible=False), "eligible") + + def test_a_verifier_error(self) -> None: + self.assert_refused(request_for(verifiers=with_reachability(raising)), "VERIFIER_ERROR") + + def test_a_failing_structural_gate(self) -> None: + real = __import__("core.verification.engine", fromlist=["run_work_set"]).run_work_set + + def broken(work: Any, inputs: Any, **kwargs: Any) -> Any: + return real(work, dataclasses.replace(inputs, candidate_state_hash="0" * 64), **kwargs) + + with mock.patch("core.application.baseline.run_work_set", broken): + self.assert_refused(request_for(), "STRUCTURAL") + + def test_an_unparseable_directory(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + (Path(tmp) / "main.tf").write_text('resource "aws_instance" {{{\n', encoding="utf-8") + self.assert_refused(request_for(Path(tmp)), "normalization") + + def test_an_empty_directory(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + self.assert_refused(request_for(Path(tmp)), "no configuration files") + + def test_a_repeat_after_a_refusal_stores_nothing_new(self) -> None: + request = request_for(eligible=False) + with self.assertRaises(BaselineRefused): + self.establish(request) + before = dump(self.path) + with self.assertRaises(BaselineRefused): + self.establish(request) + self.assertEqual(dump(self.path), before) + + def test_a_refusal_for_another_reason_is_not_a_conflict(self) -> None: + with self.assertRaises(BaselineRefused): + self.establish(request_for(eligible=False)) + with self.assertRaises(BaselineRefused): + self.establish(request_for(verifiers=with_reachability(raising))) + self.assertEqual(self.count("evidence_events"), 2) + + def test_a_refused_baseline_can_be_established_once_the_cause_is_gone(self) -> None: + with self.assertRaises(BaselineRefused): + self.establish(request_for(eligible=False)) + result = self.establish(request_for()) + self.assertEqual(result.state.version, 0) + + +class TestUncertaintyIsRecordedAsUncertain(BaselineCase): + def test_an_unknown_result_gives_an_uncertain_reference_never_protected(self) -> None: + result = self.establish(request_for(verifiers=with_reachability(unknown))) + by = {r.invariant_id: r.status for r in result.state.invariant_refs} + self.assertIs(by[FUNC], InvariantStatus.UNCERTAIN) + self.assertIs(by[SEC1], InvariantStatus.VIOLATED) + self.assertIs(by[SEC2], InvariantStatus.PROTECTED) + + +class TestTheWriteIsAtomic(BaselineCase): + POINTS = ( + "after_trusted_state_insert", + "after_invariant_refs_insert", + "after_lineage_head_update", + "before_commit", + ) + + def test_a_failure_at_each_checkpoint_leaves_nothing(self) -> None: + for point in self.POINTS: + with self.subTest(point=point): + gate = Gate(point, fail=True) + gate.release.set() + with self.assertRaises(Exception) as caught: + self.establish(checkpoint=gate) + self.assertTrue(gate.fired) + self.assertTrue( + isinstance(caught.exception, InjectedFault) + or isinstance(caught.exception.__cause__, InjectedFault) + or "InjectedFault" in repr(caught.exception), + repr(caught.exception), + ) + self.assertEqual( + self.tables(), + { + "trusted_states": 0, + "lineage_heads": 0, + "invariants": 0, + "invariant_refs": 0, + "evidence_events": 0, + "audit_events": 0, + }, + ) + + def test_after_a_failure_the_same_request_succeeds(self) -> None: + gate = Gate("after_lineage_head_update", fail=True) + gate.release.set() + with self.assertRaises((InjectedFault, PersistenceError)): + self.establish(checkpoint=gate) + result = self.establish() + self.assertEqual(result.state.version, 0) + self.assertEqual(self.count("lineage_heads"), 1) + + +class TestArguments(BaselineCase): + def test_wrong_types_are_refused(self) -> None: + with self.assertRaises(DomainValidationError): + establish_baseline(self.factory(), {"baseline_id": BASELINE_ID}) # type: ignore[arg-type] + with self.assertRaises(DomainValidationError): + establish_baseline("not a factory", request_for()) # type: ignore[arg-type] + for bad in ( + {"baseline_id": "x"}, + {"lineage_id": "x"}, + {"eligible": "yes"}, + {"definitions": ()}, + {"baseline_dir": "fixtures"}, + ): + with self.subTest(bad=bad), self.assertRaises(DomainValidationError): + self.establish(request_for(**bad)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_chain.py b/tests/unit/test_chain.py new file mode 100644 index 0000000..915ebdc --- /dev/null +++ b/tests/unit/test_chain.py @@ -0,0 +1,483 @@ +"""The evidence chain and the checker that defines "complete" (P6c step 2; Doc 11 §12, §43, §44, +§51.3; Doc 06 §28; C-172). + +HAND EXPECTATIONS (the order of C-172 and Doc 11 §51.3, not the code's output): + +* a safe promotion: CANDIDATE_CREATED, IDENTITY_COMPLETED, DEPENDENCY_COMPLETED, IMPACT_COMPLETED, + one VERIFICATION_COMPLETED per result, PROMOTION_DECIDED, PROMOTION_REQUESTED, STATE_PROMOTED; + its candidate-bound evidence is STATE, IDENTITY, DEPENDENCY, IMPACT, then one VERIFICATION per + result; a binding record is bound to the new state; +* a rejected candidate: the same up to PROMOTION_DECIDED, then CANDIDATE_REJECTED, and no + STATE_PROMOTED; +* ``problems()`` is empty for both, and fires, with the name of the rule, for each rule of C-172 + that a hand-broken database violates: the order, a named record that is missing, an integrity + failure, a record bound to the wrong candidate or hash, an incomplete decision, a promotion + without STATE_PROMOTED or its binding record, a rejection with STATE_PROMOTED or without its + outcome event, a stale decision whose parent is still current, a FAILED candidate without + ERROR_OCCURRED. +""" + +from __future__ import annotations + +import unittest +from typing import Any + +from core.application import ids as oids +from core.application.chain import describe_chain +from core.domain.audit import AuditEventType +from core.domain.enums import CandidateStatus, DecisionType +from core.domain.evidence import EvidenceKind +from core.domain.promotion import DecisionReason +from core.domain.state import CandidateState, fail_building, start_building +from core.promotion import ids as pids +from tests.domain_builders import patch_for, uid +from tests.identity_builders import NOW as PIPELINE_NOW +from tests.promotion_commit_builders import NOW +from tests.promotion_e2e_builders import SAFE, ChainCase, run_for + +T = AuditEventType +K = EvidenceKind +PREFIX = [T.CANDIDATE_CREATED, T.IDENTITY_COMPLETED, T.DEPENDENCY_COMPLETED, T.IMPACT_COMPLETED] + + +def codes(problems: list[str]) -> set[str]: + return {p.split(":", 1)[0] for p in problems} + + +class ChainReader(ChainCase): + def chain(self, candidate_id: str) -> Any: + with self.uow() as u: + return describe_chain(u, candidate_id) + + def failed(self, *, with_error: bool) -> CandidateState: + run = run_for(SAFE) + self.install(run) + trusted = run.trusted + created = CandidateState.create( + parent=trusted, + patch=patch_for(trusted, n=21), + candidate_sequence=5, + now=PIPELINE_NOW, + candidate_id=uid(3500), + ) + building = start_building(created) + failed = fail_building(building, "the configuration does not parse") + with self.uow() as u: + u.patches.save(patch_for(trusted, n=21)) + u.candidates.create(created) + u.candidates.save_transition(building) + u.candidates.save_transition(failed) + self.recorder.emit( + u, + T.CANDIDATE_CREATED, + event_id=oids.event_id_for(failed.candidate_id, 1, T.CANDIDATE_CREATED), + payload={ + "parent_state_id": trusted.state_id, + "patch_id": failed.patch_id, + "source": failed.source.value, + }, + when=NOW, + candidate_id=failed.candidate_id, + ) + if with_error: + self.recorder.record_error( + u, + stage="normalize", + error_type="ConstructionFailed", + detail={"status": "INVALID"}, + attempt_number=1, + at=PIPELINE_NOW, + when=NOW, + candidate=failed, + ) + return failed + + +class TestAGoodChain(ChainReader): + def test_a_safe_promotion_is_complete_and_in_the_order_of_c_172(self) -> None: + run = self.promoted_chain() + chain = self.chain(run.candidate.candidate_id) + n = len(run.verification.results) + self.assertEqual( + chain.event_types(), + [e.value for e in PREFIX] + + ["VERIFICATION_COMPLETED"] * n + + ["PROMOTION_DECIDED", "PROMOTION_REQUESTED", "STATE_PROMOTED"], + ) + self.assertEqual( + chain.evidence_kinds(), + ["STATE", "IDENTITY", "DEPENDENCY", "IMPACT"] + ["VERIFICATION"] * n, + ) + self.assertEqual(chain.problems(), []) + self.assertIsNotNone(chain.binding) + assert chain.state is not None and chain.binding is not None + self.assertEqual(chain.binding.state_id, chain.state.state_id) + self.assertEqual(chain.binding.state_hash, chain.state.state_hash) + self.assertIs(chain.candidate.status, CandidateStatus.PROMOTED) + + def test_a_rejected_candidate_is_complete_and_has_no_state_promoted(self) -> None: + run = self.rejected_chain() + chain = self.chain(run.candidate.candidate_id) + n = len(run.verification.results) + self.assertEqual( + chain.event_types(), + [e.value for e in PREFIX] + + ["VERIFICATION_COMPLETED"] * n + + ["PROMOTION_DECIDED", "CANDIDATE_REJECTED"], + ) + self.assertEqual(chain.problems(), []) + self.assertIsNone(chain.state) + self.assertIs(chain.candidate.status, CandidateStatus.REJECTED) + + def test_an_unfinished_chain_has_no_problem_as_long_as_it_is_a_prefix(self) -> None: + run = self.promoted_chain(fixture_id=SAFE) + del run + run2 = run_for(SAFE, candidate_id=uid(3998), sequence=2, patch_n=8) + self.patch_numbers[run2.candidate.candidate_id] = 8 + self.store(run2, sequence=2, patch_n=8) + self.emit_created(run2) + for stage in ("state", "identity", "dependency"): + self.record_stage(run2, stage) + chain = self.chain(run2.candidate.candidate_id) + self.assertIs(chain.candidate.status, CandidateStatus.ANALYZING) + self.assertEqual(chain.problems(finished=False), []) + self.assertTrue(chain.problems(finished=True)) + + def test_an_unknown_candidate_is_a_problem(self) -> None: + chain = self.chain(uid(123456)) + self.assertIn("UNREADABLE", codes(chain.problems())) + + +class TestARecordedError(ChainReader): + """An error that was recorded and then recovered from is history, not a broken chain.""" + + def error_after_the_end(self, run: Any) -> None: + with self.uow() as u: + self.recorder.record_error( + u, + stage="commit", + error_type="InjectedFault", + detail={"message": "a read after the promotion failed"}, + attempt_number=1, + at=PIPELINE_NOW, + when=NOW, + candidate=run.candidate, + evidence_id=oids.evidence_id_for(run.candidate.candidate_id, 1, "error", "k"), + event_id=oids.event_id_for(run.candidate.candidate_id, 1, T.ERROR_OCCURRED, "k"), + ) + + def test_an_error_after_a_promotion_is_not_a_problem(self) -> None: + run = self.promoted_chain() + self.error_after_the_end(run) + chain = self.chain(run.candidate.candidate_id) + self.assertEqual(chain.event_types()[-2:], ["STATE_PROMOTED", "ERROR_OCCURRED"]) + self.assertEqual(chain.problems(), []) + + def test_an_error_after_a_rejection_is_not_a_problem(self) -> None: + run = self.rejected_chain() + self.error_after_the_end(run) + self.assertEqual(self.chain(run.candidate.candidate_id).problems(), []) + + def test_a_failed_candidate_is_still_built_around_its_error(self) -> None: + """The leniency stops at a FAILED candidate: without ERROR_OCCURRED it is a problem.""" + failed = self.failed(with_error=False) + self.assertIn("ENDING", codes(self.chain(failed.candidate_id).problems())) + + +class TestTheOrder(ChainReader): + def test_stages_recorded_out_of_order_are_found(self) -> None: + run = run_for(SAFE) + self.install(run) + self.store(run) + self.write_chain(run, order=("state", "impact", "identity", "dependency", "verification")) + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("ORDER", codes(problems)) + + def test_a_missing_stage_is_found(self) -> None: + run = run_for(SAFE) + self.install(run) + self.store(run) + self.write_chain(run, stages=("state", "identity", "impact", "verification")) + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("ORDER", codes(problems)) + + def test_a_missing_verification_is_found(self) -> None: + run = run_for(SAFE) + self.install(run) + self.store(run) + self.write_chain(run, stages=("state", "identity", "dependency", "impact")) + self.assertIn("ORDER", codes(self.chain(run.candidate.candidate_id).problems())) + + def test_the_evidence_kinds_must_follow_the_stages(self) -> None: + run = run_for(SAFE) + self.install(run) + self.store(run) + self.write_chain(run, order=("identity", "state", "dependency", "impact", "verification")) + self.assertIn("EVIDENCE-ORDER", codes(self.chain(run.candidate.candidate_id).problems())) + + +class TestNamedEvidence(ChainReader): + def test_an_event_naming_a_record_that_does_not_exist_is_found(self) -> None: + run = self.promoted_chain() + identity = oids.evidence_id_for(run.candidate.candidate_id, 1, "identity") + self.tamper("DELETE FROM evidence_events WHERE evidence_id = ?", (identity,)) + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("NAMED-EVIDENCE", codes(problems)) + + def test_a_verification_record_no_event_names_is_found(self) -> None: + run = self.promoted_chain() + self.tamper( + "DELETE FROM audit_events WHERE event_type = 'VERIFICATION_COMPLETED' " + "AND rowid = (SELECT MIN(rowid) FROM audit_events " + "WHERE event_type = 'VERIFICATION_COMPLETED')" + ) + problems = self.chain(run.candidate.candidate_id).problems() + self.assertTrue({"NAMED-EVIDENCE", "ORDER"} & codes(problems), problems) + + +class TestIntegrity(ChainReader): + def test_a_tampered_payload_is_found(self) -> None: + run = self.promoted_chain() + impact = oids.evidence_id_for(run.candidate.candidate_id, 1, "impact") + with self.uow() as u: + row = u.evidence.get(impact) + assert row is not None + digest = row.content_hash + self.tamper( + "UPDATE evidence_artifacts SET payload_json = ? WHERE content_hash = ?", + ('{"tampered":true}', digest), + ) + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("INTEGRITY", codes(problems)) + + def test_a_missing_payload_is_found(self) -> None: + run = self.promoted_chain() + impact = oids.evidence_id_for(run.candidate.candidate_id, 1, "impact") + with self.uow() as u: + row = u.evidence.get(impact) + assert row is not None + self.tamper("DELETE FROM evidence_artifacts WHERE content_hash = ?", (row.content_hash,)) + self.assertIn("INTEGRITY", codes(self.chain(run.candidate.candidate_id).problems())) + + +class TestBinding(ChainReader): + def test_a_record_with_another_state_hash_is_found(self) -> None: + run = self.promoted_chain() + identity = oids.evidence_id_for(run.candidate.candidate_id, 1, "identity") + # the repository refuses a column that disagrees with the stored record (DATA-INT-010), so + # the column and the record are changed together, as an attacker with file access would + self.tamper( + "UPDATE evidence_events SET state_hash = ?, " + "content_json = json_set(content_json, '$.state_hash', ?) WHERE evidence_id = ?", + ("c" * 64, "c" * 64, identity), + ) + self.assertIn("BINDING", codes(self.chain(run.candidate.candidate_id).problems())) + + def test_a_record_with_another_parent_is_found(self) -> None: + run = self.promoted_chain() + dependency = oids.evidence_id_for(run.candidate.candidate_id, 1, "dependency") + self.tamper( + "UPDATE evidence_events SET parent_state_id = ?, " + "content_json = json_set(content_json, '$.parent_state_id', ?) WHERE evidence_id = ?", + (run.candidate.candidate_id, run.candidate.candidate_id, dependency), + ) + self.assertIn("BINDING", codes(self.chain(run.candidate.candidate_id).problems())) + + +class TestTheDecisionRow(ChainReader): + def test_a_terminal_candidate_without_a_decision_is_found(self) -> None: + run = self.rejected_chain() + self.tamper("DELETE FROM promotion_decisions") + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("DECISION", codes(problems)) + + def test_a_decision_supported_by_a_result_that_is_not_in_the_chain_is_found(self) -> None: + run = self.rejected_chain() + did = oids.decision_id_for(run.candidate.candidate_id, 1) + with self.uow() as u: + row = u.decisions.get(did) + assert row is not None + # a result id that no VERIFICATION record of the candidate carries + stray = uid(7777777) + self.tamper( + "UPDATE promotion_decisions SET supporting_results_json = ?, content_hash = ? " + "WHERE decision_id = ?", + self.with_supporting(row, [stray]), + ) + self.assertIn("DECISION", codes(self.chain(run.candidate.candidate_id).problems())) + + def with_supporting(self, row: Any, supporting: list[str]) -> tuple[Any, ...]: + """The SQL parameters that give ``row`` another list of supporting results, with the hash + the domain computes for that content (so the row still reads back).""" + import json + + from core.domain.promotion import PromotionDecision + + changed = PromotionDecision.create( + decision_id=row.decision_id, + candidate_id=row.candidate_id, + parent_state_id=row.parent_state_id, + decision=row.decision, + reason=row.reason, + policy_version=row.policy_version, + attempt_number=row.attempt_number, + required_checks=row.required_checks, + blocking_conditions=row.blocking_conditions, + supporting_results=supporting, + created_at=row.created_at, + ) + return (json.dumps(list(supporting)), changed.content_hash, row.decision_id) + + def test_a_decision_with_a_required_check_outside_its_supporting_results_is_found( + self, + ) -> None: + run = self.rejected_chain() + did = oids.decision_id_for(run.candidate.candidate_id, 1) + with self.uow() as u: + row = u.decisions.get(did) + assert row is not None + # keep the rows consistent with their hash, but drop the supporting results + self.tamper( + "UPDATE promotion_decisions SET supporting_results_json = ?, content_hash = ? " + "WHERE decision_id = ?", + self.with_supporting(row, []), + ) + self.assertIn("DECISION", codes(self.chain(run.candidate.candidate_id).problems())) + + +class TestTheEnding(ChainReader): + def test_a_promotion_without_state_promoted_is_found(self) -> None: + run = self.promoted_chain() + self.tamper("DELETE FROM audit_events WHERE event_type = 'STATE_PROMOTED'") + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("ENDING", codes(problems)) + + def test_a_promotion_without_its_binding_record_is_found(self) -> None: + run = self.promoted_chain() + with self.uow() as u: + did = u.decisions.for_candidate(run.candidate.candidate_id)[0].decision_id + self.tamper( + "DELETE FROM evidence_events WHERE evidence_id = ?", (pids.evidence_id_for(did),) + ) + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("PROMOTED-STATE", codes(problems)) + + def test_a_binding_record_for_another_state_hash_is_found(self) -> None: + run = self.promoted_chain() + with self.uow() as u: + did = u.decisions.for_candidate(run.candidate.candidate_id)[0].decision_id + self.tamper( + "UPDATE evidence_events SET state_hash = ? WHERE evidence_id = ?", + ("d" * 64, pids.evidence_id_for(did)), + ) + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("PROMOTED-STATE", codes(problems)) + + def test_a_rejection_without_its_outcome_event_is_found(self) -> None: + run = self.rejected_chain() + self.tamper("DELETE FROM audit_events WHERE event_type = 'CANDIDATE_REJECTED'") + self.assertIn("ENDING", codes(self.chain(run.candidate.candidate_id).problems())) + + def test_a_rejection_with_a_state_promoted_event_is_found(self) -> None: + run = self.rejected_chain() + with self.uow() as u: + decision = u.decisions.for_candidate(run.candidate.candidate_id)[0] + self.recorder.emit( + u, + T.STATE_PROMOTED, + event_id=uid(8881), + payload={"parent_state_id": run.trusted.state_id}, + when=NOW, + state_id=run.trusted.state_id, + candidate_id=run.candidate.candidate_id, + decision_id=decision.decision_id, + ) + problems = self.chain(run.candidate.candidate_id).problems() + self.assertTrue({"ENDING", "ORDER"} & codes(problems), problems) + + def test_a_candidate_marked_promoted_without_a_committed_decision_is_found(self) -> None: + run = self.promoted_chain() + self.tamper("UPDATE promotion_decisions SET committed_state_id = NULL") + problems = self.chain(run.candidate.candidate_id).problems() + self.assertIn("DECISION", codes(problems) | codes(self.chain_problems_unfinished(run))) + + def chain_problems_unfinished(self, run: Any) -> list[str]: + return self.chain(run.candidate.candidate_id).problems(finished=False) + + +class TestAStaleChain(ChainReader): + def stale(self) -> Any: + first = self.promoted_chain() + second = self.second_candidate() + self.write_chain(second) + return first, second + + def test_a_stale_candidate_ends_with_a_stale_parent_rejection(self) -> None: + _, second = self.stale() + chain = self.chain(second.candidate.candidate_id) + self.assertEqual(chain.problems(), []) + self.assertIs(chain.candidate.status, CandidateStatus.REJECTED) + [decision] = [d for d in chain.decisions if d.reason is DecisionReason.STALE_PARENT] + self.assertIs(decision.decision, DecisionType.REJECT) + self.assertEqual(chain.event_types()[-2:], ["PROMOTION_DECIDED", "CANDIDATE_REJECTED"]) + + def test_a_stale_rejection_whose_parent_is_still_current_is_found(self) -> None: + """REJECT/STALE_PARENT is stored for a candidate of the current state: not stale.""" + run = run_for(SAFE) + self.install(run) + self.store(run) + self.emit_created(run) + for stage in ("state", "identity", "dependency", "impact", "verification"): + self.record_stage(run, stage) + from core.promotion.commit import record_decision + from core.promotion.decide import decide + from tests.promotion_commit_builders import CONTEXT + + outcome = decide( + run.candidate, + run.newer_trusted(), # another state of the lineage: the candidate is stale for decide + run.impact, + run.verification, + run.policy(), + decision_id=oids.decision_id_for(run.candidate.candidate_id, 1), + now=PIPELINE_NOW, + ) + self.assertIs(outcome.decision.reason, DecisionReason.STALE_PARENT) + with self.uow() as u: + record_decision(u, outcome, run.candidate, CONTEXT, now=NOW) + # nothing was promoted: the candidate's parent is still the lineage head + self.assertIn("STALE", codes(self.chain(run.candidate.candidate_id).problems())) + + +class TestAFailedCandidate(ChainReader): + def test_a_failed_candidate_with_its_error_record_is_complete(self) -> None: + failed = self.failed(with_error=True) + chain = self.chain(failed.candidate_id) + self.assertEqual(chain.event_types(), ["CANDIDATE_CREATED", "ERROR_OCCURRED"]) + self.assertEqual(chain.problems(), []) + self.assertEqual(chain.decisions, ()) + self.assertIsNone(chain.state) + + def test_a_failed_candidate_without_error_occurred_is_found(self) -> None: + failed = self.failed(with_error=False) + problems = self.chain(failed.candidate_id).problems() + self.assertIn("ENDING", codes(problems)) + + +class TestTheChainReadsOnlyThisCandidate(ChainReader): + def test_another_candidates_records_are_not_in_the_chain(self) -> None: + first = self.promoted_chain() + second = self.second_candidate() + self.write_chain(second) + one = self.chain(first.candidate.candidate_id) + two = self.chain(second.candidate.candidate_id) + self.assertTrue(all(e.candidate_id == first.candidate.candidate_id for e in one.evidence)) + self.assertTrue(all(e.candidate_id == second.candidate.candidate_id for e in two.evidence)) + self.assertTrue( + {K.STATE, K.IDENTITY, K.DEPENDENCY, K.IMPACT, K.VERIFICATION} + >= {e.kind for e in one.evidence} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_e2e_chains.py b/tests/unit/test_e2e_chains.py new file mode 100644 index 0000000..a7e3295 --- /dev/null +++ b/tests/unit/test_e2e_chains.py @@ -0,0 +1,213 @@ +"""The evidence chains of Doc 11 §51.3, end to end (P6c step 5; Doc 11 §12, §41-§44, §51.3; Doc 06 +§27-§28; Doc 09 §20-§25; C-57, C-170, C-172, C-173). + +Each test runs the real orchestrator on fixtures, reads the chain back with ``describe_chain``, +asserts that ``problems() == []`` and asserts the exact ordered event types and the evidence kinds. + +HAND EXPECTATIONS (Doc 11 §41-§44, the C-172 order, the fixtures): + +* every evaluated candidate: CANDIDATE_CREATED, IDENTITY_COMPLETED, DEPENDENCY_COMPLETED, + IMPACT_COMPLETED, one VERIFICATION_COMPLETED per result; its bound evidence is STATE, IDENTITY, + DEPENDENCY, IMPACT and one VERIFICATION record per result; +* then: safe promotion PROMOTION_DECIDED, PROMOTION_REQUESTED, STATE_PROMOTED; rejection + PROMOTION_DECIDED, CANDIDATE_REJECTED; retry PROMOTION_DECIDED, RETRY_SCHEDULED; escalation + PROMOTION_DECIDED, ESCALATED; +* the stale parent chain has the same stages (the stages ran against the recorded parent, C-173) and + ends as a rejection, reason STALE_PARENT; +* a construction failure is CANDIDATE_CREATED, ERROR_OCCURRED with no decision; +* retry: ``func_unknown_dependency_unknown_v1`` is RETRY at the first attempt; the same problem at + the policy's attempt budget (3) is an ESCALATE ``RETRY_BUDGET_EXHAUSTED``. +""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from typing import Any + +from core.application.chain import EvidenceChain, describe_chain +from core.application.orchestrator import EvaluationResult, ResultKind +from core.domain.enums import CandidateStatus, DecisionType +from core.domain.evidence import EvidenceKind +from core.domain.promotion import DecisionReason +from core.verification.dispatch import DEFAULT_VERIFIERS, REACHABILITY_CFG +from tests.promotion_orchestrator_builders import ( + PATCH_REGRESSION_DIR, + PATCH_SAFE_DIR, + OrchestratorCase, + config_with, +) +from tests.promotion_pipeline import ROOT, failing_reachability, fixtures_with_a_parent + +HEAD = ["CANDIDATE_CREATED", "IDENTITY_COMPLETED", "DEPENDENCY_COMPLETED", "IMPACT_COMPLETED"] +KINDS = ["STATE", "IDENTITY", "DEPENDENCY", "IMPACT"] + + +class ChainCase(OrchestratorCase): + def chain(self, result: EvaluationResult) -> EvidenceChain: + with self.uow() as u: + return describe_chain(u, result.candidate.candidate_id) + + def assert_chain(self, result: EvaluationResult, tail: list[str]) -> EvidenceChain: + """The chain of a candidate that was evaluated: no problem, the exact order.""" + assert result.verification is not None + n = len(result.verification.results) + chain = self.chain(result) + self.assertEqual(chain.problems(), []) + self.assertEqual(chain.event_types(), [*HEAD, *["VERIFICATION_COMPLETED"] * n, *tail]) + self.assertEqual(chain.evidence_kinds(), [*KINDS, *["VERIFICATION"] * n]) + return chain + + def fixture(self, fixture_id: str) -> Any: + return fixtures_with_a_parent()[fixture_id].data + + +class TestSafePromotion(ChainCase): + def test_candidate_identity_dependency_impact_verification_decision_state(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_SAFE_DIR)) + self.assertIs(result.kind, ResultKind.PROMOTED) + chain = self.assert_chain( + result, ["PROMOTION_DECIDED", "PROMOTION_REQUESTED", "STATE_PROMOTED"] + ) + assert chain.state is not None and chain.binding is not None + self.assertEqual(chain.state, result.state) + self.assertEqual(chain.binding.state_id, chain.state.state_id) + self.assertIs(chain.candidate.status, CandidateStatus.PROMOTED) + + +class TestRejectedCandidate(ChainCase): + def test_impact_evidence_verification_decision_reject_and_the_state_unchanged(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_REGRESSION_DIR)) + self.assertIs(result.kind, ResultKind.REJECTED) + chain = self.assert_chain(result, ["PROMOTION_DECIDED", "CANDIDATE_REJECTED"]) + self.assertIsNone(chain.state) + self.assertEqual(self.current(), v0) + # Doc 11 §42: verification SEC PASS, FUNC FAIL + assert result.verification is not None + by = {r.target_id: r.result.value for r in result.verification.results} + self.assertEqual(by["INVARIANT:INV-SEC-001:1"], "PASS") + self.assertEqual(by["INVARIANT:INV-FUNC-001:1"], "FAIL") + + +class TestRetry(ChainCase): + FIXTURE = "func_unknown_dependency_unknown_v1" + + def test_retry_then_the_budget_ends_in_an_escalation(self) -> None: + data = self.fixture(self.FIXTURE) + v0 = self.baseline(ROOT / data["parent"]).state + directory = ROOT / data["config"] + first = self.evaluate(self.request(v0, directory, n=1, attempt=1)) + self.assertIs(first.kind, ResultKind.RETRY_REQUIRED) + self.assertIs(first.candidate.status, CandidateStatus.RETRY_REQUIRED) + self.assert_chain(first, ["PROMOTION_DECIDED", "RETRY_SCHEDULED"]) + self.assertEqual(self.current(), v0) + second = self.evaluate(self.request(v0, directory, n=2, attempt=3)) + self.assertIs(second.kind, ResultKind.ESCALATED) + assert second.decision is not None + self.assertIs(second.decision.decision, DecisionType.ESCALATE) + self.assertIs(second.decision.reason, DecisionReason.RETRY_BUDGET_EXHAUSTED) + self.assert_chain(second, ["PROMOTION_DECIDED", "ESCALATED"]) + self.assertEqual(self.current(), v0) + self.assertEqual(self.count("trusted_states"), 1) + + +class TestEscalationByVerifierError(ChainCase): + FIXTURE = "process_verifier_error_escalate_v1" + + def test_a_verifier_error_is_an_escalation_and_nothing_is_promoted(self) -> None: + data = self.fixture(self.FIXTURE) + v0 = self.baseline(ROOT / data["parent"]).state + table = {**DEFAULT_VERIFIERS, REACHABILITY_CFG: failing_reachability} + result = self.evaluate(self.request(v0, ROOT / data["config"]), verifiers=table) + self.assertIs(result.kind, ResultKind.ESCALATED) + assert result.decision is not None + self.assertIs(result.decision.reason, DecisionReason.VERIFIER_ERROR) + self.assertIn( + "VERIFIER_ERROR", + {r.result.value for r in (result.verification.results if result.verification else ())}, + ) + self.assert_chain(result, ["PROMOTION_DECIDED", "ESCALATED"]) + self.assertEqual(self.current(), v0) + self.assertEqual(data["decision"]["allowed"], ["ESCALATE"]) + + +class TestStaleParent(ChainCase): + def test_the_stale_candidate_has_the_whole_chain_and_ends_as_a_stale_rejection(self) -> None: + v0 = self.baseline().state + stale_request = self.request(v0, PATCH_SAFE_DIR, n=2) + winner = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=1)) + assert winner.state is not None + stale = self.evaluate(stale_request) + self.assertIs(stale.kind, ResultKind.STALE) + chain = self.assert_chain(stale, ["PROMOTION_DECIDED", "CANDIDATE_REJECTED"]) + [decision] = chain.decisions + self.assertIs(decision.reason, DecisionReason.STALE_PARENT) + self.assertIs(decision.decision, DecisionType.REJECT) + self.assertEqual(decision.parent_state_id, v0.state_id) + self.assertEqual(self.current(), winner.state) + self.assertEqual(chain.head_state_id, winner.state.state_id) + self.assertEqual(self.count("trusted_states"), 2) + + def test_the_process_fixture_describes_this_case(self) -> None: + data = self.fixture("process_stale_parent_reject_v1") + self.assertEqual(data["decision"]["allowed"], ["REJECT"]) + self.assertEqual(data["process"]["kind"], "stale_parent") + + +class TestConstructionFailure(ChainCase): + def test_an_unparseable_directory_is_a_failed_candidate_and_an_error(self) -> None: + v0 = self.baseline().state + tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + (tmp / "main.tf").write_text('resource "aws_instance" "x" {{{\n', encoding="utf-8") + request = self.request(v0, tmp) + result = self.evaluate(request) + self.assertIs(result.kind, ResultKind.CONSTRUCTION_FAILED) + chain = self.chain(result) + self.assertEqual(chain.problems(), []) + self.assertEqual(chain.event_types(), ["CANDIDATE_CREATED", "ERROR_OCCURRED"]) + self.assertEqual(chain.evidence_kinds(), []) + self.assertEqual(chain.decisions, ()) + self.assertIs(chain.candidate.status, CandidateStatus.FAILED) + with self.uow() as u: + errors = [ + e + for e in u.evidence.list_for_correlation(request.context.correlation_id) + if e.kind is EvidenceKind.ERROR + ] + self.assertEqual(len(errors), 1) + self.assertEqual(self.current(), v0) + + +class TestEmergencySwitch(ChainCase): + automatic_enabled = False + + def test_a_would_be_promote_is_an_audited_escalation(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_SAFE_DIR)) + self.assertIs(result.kind, ResultKind.ESCALATED) + assert result.decision is not None + self.assertIs(result.decision.reason, DecisionReason.AUTOMATIC_PROMOTION_DISABLED) + chain = self.assert_chain(result, ["PROMOTION_DECIDED", "ESCALATED"]) + self.assertIsNone(chain.state) + self.assertEqual(self.current(), v0) + self.assertEqual(self.count("trusted_states"), 1) + + def test_the_same_candidate_is_promoted_with_the_switch_on(self) -> None: + """The switch is the only difference: the pipeline and its evidence are the same.""" + v0 = self.baseline().state + off = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=1)) + on = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=2), config=config_with(True)) + self.assertIs(off.kind, ResultKind.ESCALATED) + self.assertIs(on.kind, ResultKind.PROMOTED) + assert off.outcome is not None and on.outcome is not None + self.assertEqual( + [(e.invariant_id, e.source) for e in off.outcome.proof_plan], + [(e.invariant_id, e.source) for e in on.outcome.proof_plan], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_e2e_concurrency.py b/tests/unit/test_e2e_concurrency.py new file mode 100644 index 0000000..ea423e0 --- /dev/null +++ b/tests/unit/test_e2e_concurrency.py @@ -0,0 +1,203 @@ +"""Two orchestrators on one database file (P6c step 5; Doc 09 §25, §39; Doc 12 §14 steps 10-11; +C-38, C-166, C-173; R5 of the P6c prompt). + +Two candidates, both written against version 0, are evaluated by two ``EvaluationOrchestrator`` +objects on two threads over one SQLite file. A ``Schedule`` decides in which order the threads pass +the stage checkpoints of the orchestrator (the test-only ``checkpoint`` hook), so every interleaving +is forced by the schedule, not by timing: nothing sleeps. + +HAND EXPECTATIONS (Doc 09 §25, §39; CLAUDE.md canonical scenario; C-166): + +* exactly one of the two is promoted, into version 1, parent version 0; the other is REJECTED with + ``STALE_PARENT`` (kind STALE) and its candidate is REJECTED; Trusted State has exactly two states; +* a loser that read the state before the winner committed loses inside the commit (the PROMOTE it + recorded stays beside the stored REJECT); a loser that started after the winner committed is + rejected before any commit; both chains are complete (``problems() == []``); +* the structural rules of promoted history hold (``check_database``): one child per parent, the + head is the newest state, every promoted state has its decision and its binding record. + +The "schedules" are the deterministic ones named below and 24 random permutations seeded with the +numbers in ``SEEDS`` (the failing seed is in the subtest label). +""" + +from __future__ import annotations + +import random +import threading +import unittest +from typing import Any + +from core.application.chain import describe_chain +from core.application.orchestrator import EvaluationResult, ResultKind +from core.domain.enums import CandidateStatus, DecisionType +from core.domain.promotion import DecisionReason +from tests.promotion_commit_builders import Runner, check_database +from tests.promotion_orchestrator_builders import PATCH_SAFE_DIR, OrchestratorCase + +STAGE_POINTS = 8 # the checkpoints one full evaluation passes (a stale loser passes 7) +SEEDS = tuple(range(7000, 7024)) +WAIT = 60.0 + + +class Schedule: + """The order in which two threads pass their checkpoints. ``order`` holds ``"A"`` and ``"B"`` + (a thread may pass its next checkpoint when its letter is first) and ``"A_done"``/``"B_done"`` + (consumed once that thread has finished). A thread whose partner has finished never waits.""" + + def __init__(self, order: list[str]) -> None: + self.order = list(order) + self.done: set[str] = set() + self.cv = threading.Condition() + + def _ready(self, who: str) -> bool: + other = "B" if who == "A" else "A" + while self.order and self.order[0].endswith("_done") and self.order[0][0] in self.done: + self.order.pop(0) + return not self.order or self.order[0] == who or other in self.done + + def hook(self, who: str) -> Any: + def passage(name: str) -> None: + del name + with self.cv: + if not self.cv.wait_for(lambda: self._ready(who), timeout=WAIT): + raise TimeoutError(f"{who} waited for its turn: {self.order[:6]}") + if self.order and self.order[0] == who: + self.order.pop(0) + self.cv.notify_all() + + return passage + + def finished(self, who: str) -> None: + with self.cv: + self.done.add(who) + self.cv.notify_all() + + +def random_order(seed: int) -> list[str]: + order = ["A"] * STAGE_POINTS + ["B"] * STAGE_POINTS + random.Random(seed).shuffle(order) + return order + + +class RaceCase(OrchestratorCase): + def race(self, order: list[str]) -> tuple[EvaluationResult, EvaluationResult]: + v0 = self.baseline().state + schedule = Schedule(order) + requests = { + "A": self.request(v0, PATCH_SAFE_DIR, n=1), + "B": self.request(v0, PATCH_SAFE_DIR, n=2), + } + + def work(who: str) -> Any: + def run() -> EvaluationResult: + try: + return self.orchestrator(checkpoint=schedule.hook(who)).evaluate(requests[who]) + finally: + schedule.finished(who) + + return run + + runners = {who: Runner(work(who)) for who in ("A", "B")} + for runner in runners.values(): + runner.start() + for runner in runners.values(): + runner.join(WAIT * 2) + self.assertFalse(runner.is_alive(), "a thread never finished") + for who, runner in runners.items(): + self.assertIsNone(runner.error, f"{who}: {runner.error!r}") + return runners["A"].result, runners["B"].result + + def assert_one_winner(self, a: EvaluationResult, b: EvaluationResult) -> None: + kinds = sorted(r.kind.value for r in (a, b)) + self.assertEqual(kinds, sorted([ResultKind.PROMOTED.value, ResultKind.STALE.value])) + winner, loser = (a, b) if a.kind is ResultKind.PROMOTED else (b, a) + assert winner.state is not None and loser.decision is not None + self.assertEqual(self.count("trusted_states"), 2) + self.assertEqual(self.current(), winner.state) + self.assertEqual(winner.state.version, 1) + self.assertIs(loser.decision.decision, DecisionType.REJECT) + self.assertIs(loser.decision.reason, DecisionReason.STALE_PARENT) + self.assertIsNone(loser.state) + self.assertEqual(self.count("promotion_decisions", "committed_state_id IS NOT NULL"), 1) + self.assertEqual(check_database(self.path), []) + with self.uow() as u: + for result in (winner, loser): + self.assertEqual( + describe_chain(u, result.candidate.candidate_id).problems(), + [], + result.kind.value, + ) + for result, status in ( + (winner, CandidateStatus.PROMOTED), + (loser, CandidateStatus.REJECTED), + ): + stored = u.candidates.get(result.candidate.candidate_id) + assert stored is not None + self.assertIs(stored.status, status) + + +class TestNamedInterleavings(RaceCase): + def test_one_runs_to_the_end_before_the_other_starts(self) -> None: + a, b = self.race(["A"] * STAGE_POINTS + ["B"] * STAGE_POINTS) + self.assert_one_winner(a, b) + self.assertIs(a.kind, ResultKind.PROMOTED) + # the late one never asked for a commit: it was rejected before any request + with self.uow() as u: + types = [ + e.event_type.value for e in u.audit.list_for_candidate(b.candidate.candidate_id) + ] + self.assertNotIn("PROMOTION_REQUESTED", types) + self.assertNotIn("STATE_PROMOTED", types) + + def test_the_other_way_round(self) -> None: + a, b = self.race(["B"] * STAGE_POINTS + ["A"] * STAGE_POINTS) + self.assert_one_winner(a, b) + self.assertIs(b.kind, ResultKind.PROMOTED) + + def test_both_read_version_zero_and_the_second_loses_inside_its_commit(self) -> None: + """Both pass every stage and both ask for the commit; B commits only after A has finished, + so B read version 0, decided PROMOTE, and loses the race inside ``commit_promotion``.""" + order = ["A", "B"] * (STAGE_POINTS - 1) + ["A", "A_done", "B"] + a, b = self.race(order) + self.assert_one_winner(a, b) + self.assertIs(a.kind, ResultKind.PROMOTED) + with self.uow() as u: + chain = describe_chain(u, b.candidate.candidate_id) + self.assertEqual( + [t for t in chain.event_types() if t.startswith("PROMOTION")], + ["PROMOTION_DECIDED", "PROMOTION_REQUESTED", "PROMOTION_DECIDED"], + ) + self.assertEqual(len(chain.decisions), 2) # its PROMOTE, and the stored REJECT beside it + + def test_both_commit_at_the_same_time(self) -> None: + order = ["A", "B"] * STAGE_POINTS + a, b = self.race(order) + self.assert_one_winner(a, b) + + +class TestSeededSchedules(RaceCase): + def test_every_schedule_ends_with_one_promotion_and_one_stale_rejection(self) -> None: + inside, before = 0, 0 + for seed in SEEDS: + with self.subTest(seed=seed): + self.fresh_database() + a, b = self.race(random_order(seed)) + self.assert_one_winner(a, b) + loser = b if a.kind is ResultKind.PROMOTED else a + with self.uow() as u: + types = [ + e.event_type.value + for e in u.audit.list_for_candidate(loser.candidate.candidate_id) + ] + if "PROMOTION_REQUESTED" in types: + inside += 1 + else: + before += 1 + # the schedules are not all alike: they reach a loser that was rejected before it asked for + # a commit, and one that lost inside the commit + self.assertGreater(inside, 0) + self.assertGreater(before, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_e2e_fixtures.py b/tests/unit/test_e2e_fixtures.py new file mode 100644 index 0000000..6107448 --- /dev/null +++ b/tests/unit/test_e2e_fixtures.py @@ -0,0 +1,102 @@ +"""Every fixture with a parent, through ``evaluate`` and through the P6a test pipeline (P6c step 5; +Doc 12 §5-§7, §14; Doc 09 Appendix A; C-22, C-45, C-170). + +The P6a pipeline (``tests/promotion_pipeline.py``) builds the trusted state from the parent's own +``expected.json`` and runs the engines by hand; ``evaluate`` establishes the parent as a baseline +through the database (the engines' own statuses) and runs the same engines. For each fixture the two +must give the same decision, the same reason, the same blocking conditions, the same required +checks and the same proof plan. A difference is a defect in one of them and is reported, not +reconciled: this test compares, it does not choose a side. + +The comparison leaves out what cannot be equal by construction: the ids (the two sides mint their +own), the detail texts of the conditions (they quote ids), and the policy version label. + +HAND EXPECTATIONS for ``kind == "transition"`` fixtures: the decision is one of the fixture's own +``decision.allowed`` values (hand-written data, C-89). +""" + +from __future__ import annotations + +import unittest +from typing import Any + +from core.application.chain import describe_chain +from core.application.orchestrator import ResultKind +from core.domain.promotion import DecisionOutcome +from tests.domain_builders import uid +from tests.promotion_commit_builders import check_database +from tests.promotion_orchestrator_builders import OrchestratorCase +from tests.promotion_pipeline import ROOT, fixtures_with_a_parent, pipeline + + +def view(outcome: DecisionOutcome) -> dict[str, Any]: + d = outcome.decision + return { + "decision": d.decision, + "reason": d.reason, + "conditions": sorted( + (c.code.value, c.decision.value, c.target) for c in d.blocking_conditions + ), + "checks": [(c.target_id, c.kind, c.source) for c in d.required_checks], + "plan": [(e.invariant_id, e.invariant_version, e.source) for e in outcome.proof_plan], + } + + +class TestTheDifferential(OrchestratorCase): + def test_there_are_eighteen_fixtures_with_a_parent(self) -> None: + """The P6c prompt says 17; the repository holds 20 fixtures, 2 of them baselines.""" + self.assertEqual(len(fixtures_with_a_parent()), 18) + + def test_evaluate_and_the_p6a_pipeline_agree_on_every_fixture(self) -> None: + seen: set[str] = set() + for fixture_id, fixture in fixtures_with_a_parent().items(): + data = fixture.data + with self.subTest(fixture=fixture_id): + self.fresh_database() + v0 = self.baseline(ROOT / data["parent"]).state + result = self.evaluate(self.request(v0, ROOT / data["config"])) + self.assertIsNotNone(result.outcome, result.kind) + assert result.outcome is not None + run = pipeline(fixture) + reference = run.decide(run.policy(automatic_enabled=True), decision_id=uid(9300)) + self.assertEqual(view(result.outcome), view(reference)) + seen.add(result.outcome.decision.decision.value) + if data["kind"] == "transition": + allowed = data["decision"]["allowed"] + self.assertIn(result.outcome.decision.decision.value, allowed) + # the comparison is not vacuous: the fixtures reach more than one kind of decision + self.assertTrue({"PROMOTE", "REJECT"} <= seen, seen) + self.assertGreaterEqual(len(seen), 3, seen) + + def test_every_chain_is_complete_and_the_database_is_sound(self) -> None: + for fixture_id, fixture in fixtures_with_a_parent().items(): + data = fixture.data + with self.subTest(fixture=fixture_id): + self.fresh_database() + v0 = self.baseline(ROOT / data["parent"]).state + result = self.evaluate(self.request(v0, ROOT / data["config"])) + with self.uow() as u: + chain = describe_chain(u, result.candidate.candidate_id) + self.assertEqual(chain.problems(), []) + self.assertEqual(check_database(self.path), []) + self.assertIsNot(result.kind, ResultKind.CONSTRUCTION_FAILED) + + +class TestReportsAreDeterministic(OrchestratorCase): + """R1 of the P6c prompt: a rerun recomputes the reports, so two runs of the pipeline on one + input must give the same reports (up to the random ids the engines draw).""" + + def test_two_runs_give_the_same_reports(self) -> None: + for fixture_id, fixture in fixtures_with_a_parent().items(): + with self.subTest(fixture=fixture_id): + a, b = pipeline(fixture), pipeline(fixture) + self.assertEqual(a.impact.content_hash, b.impact.content_hash) + self.assertEqual(a.verification.content_hash, b.verification.content_hash) + self.assertEqual(a.inputs.identity.content_hash, b.inputs.identity.content_hash) + self.assertEqual( + a.inputs.candidate_graph.graph_hash, b.inputs.candidate_graph.graph_hash + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_e2e_idempotency.py b/tests/unit/test_e2e_idempotency.py new file mode 100644 index 0000000..86210f0 --- /dev/null +++ b/tests/unit/test_e2e_idempotency.py @@ -0,0 +1,302 @@ +"""Idempotency and crash windows of ``evaluate`` (P6c step 5; Doc 09 §26, §38-§39, Appendix B; +Doc 11 §38; C-38, C-163, C-166, C-170 (4), (5), (7)). + +HAND EXPECTATIONS (the prompt's C-170 (5) and (7), P6b's contract): + +* a second ``evaluate`` with an equal request writes nothing and returns the stored outcome; a + committed promotion comes back as ``DUPLICATE`` with the same state; this holds when the second + call is made by another orchestrator whose clock reads another time (everything that is hashed + uses the candidate's own instant, so a rerun with a real clock is not a conflict); +* an exception injected after any stage leaves every record written so far valid + (``problems(finished=False) == []``, ``check_database`` clean) and does not change Trusted State; + a rerun completes and exactly one new state exists; +* a crash between ``record_decision`` and the commit leaves a PROMOTABLE candidate and the rerun + commits it; +* a failure at the commit of the n-th unit of work, for every n, is recoverable the same way; +* a failure inside the promotion's own transaction raises ``PromotionCommitError`` (P6b), leaves the + candidate PROMOTABLE and no new state, and the same request then commits. +""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from typing import Any + +from core.application.chain import describe_chain +from core.application.orchestrator import EvaluationRequest, ResultKind +from core.domain.enums import CandidateStatus +from core.domain.errors import PersistenceError +from core.promotion.commit import PromotionCommitError +from core.verification.dispatch import DEFAULT_VERIFIERS, REACHABILITY_CFG +from tests.pipeline_builders import fixed_clock +from tests.promotion_commit_builders import Gate, InjectedFault, check_database, dump +from tests.promotion_orchestrator_builders import ( + PATCH_REGRESSION_DIR, + PATCH_SAFE_DIR, + OrchestratorCase, + config_with, +) +from tests.promotion_pipeline import failing_reachability +from tests.unit.test_orchestrator import unknown + +STAGES = ( + "after_candidate", + "after_build", + "after_identity", + "after_dependency", + "after_impact", + "after_verification", + "after_decision", + "after_request", +) + + +class Stop(Exception): + """An exception put into a stage on purpose.""" + + +def stop_after(point: str) -> Any: + def hook(name: str) -> None: + if name == point: + raise Stop(point) + + return hook + + +class IdempotencyCase(OrchestratorCase): + def chain_problems(self, request: EvaluationRequest, *, finished: bool = True) -> list[str]: + """The problems of the candidate's chain; none while nothing about it has been written.""" + with self.uow() as u: + if u.candidates.get(request.candidate_id) is None: + return [] + return describe_chain(u, request.candidate_id).problems(finished=finished) + + def scenarios(self) -> list[tuple[str, EvaluationRequest, dict[str, Any]]]: + v0 = self.baseline().state + broken = Path(self.enterContext(tempfile.TemporaryDirectory())) + (broken / "main.tf").write_text('resource "aws_instance" "x" {{{\n', encoding="utf-8") + return [ + ("promote", self.request(v0, PATCH_SAFE_DIR, n=1), {}), + ("reject", self.request(v0, PATCH_REGRESSION_DIR, n=2), {}), + ( + "retry", + self.request(v0, PATCH_SAFE_DIR, n=3), + {"verifiers": {**DEFAULT_VERIFIERS, REACHABILITY_CFG: unknown}}, + ), + ( + "verifier error", + self.request(v0, PATCH_SAFE_DIR, n=4), + {"verifiers": {**DEFAULT_VERIFIERS, REACHABILITY_CFG: failing_reachability}}, + ), + ("construction failure", self.request(v0, broken, n=5), {}), + ] + + +class TestEvaluateTwice(IdempotencyCase): + def test_the_second_call_writes_nothing_and_returns_the_stored_outcome(self) -> None: + for name, request, options in self.scenarios(): + with self.subTest(scenario=name): + first = self.evaluate(request, **options) + before = dump(self.path) + second = self.evaluate(request, **options) + self.assertEqual(dump(self.path), before) + self.assertTrue(second.duplicate) + self.assertEqual(second.decision, first.decision) + self.assertEqual(second.state, first.state) + self.assertEqual(second.candidate, first.candidate) + expected = ResultKind.DUPLICATE if first.state is not None else first.kind + self.assertIs(second.kind, expected) + + def test_another_orchestrator_with_another_clock_is_still_a_duplicate(self) -> None: + for name, request, options in self.scenarios(): + with self.subTest(scenario=name): + self.evaluate(request, clock=fixed_clock(10), **options) + before = dump(self.path) + again = self.evaluate(request, clock=fixed_clock(5000), **options) + self.assertTrue(again.duplicate) + self.assertEqual(dump(self.path), before) + + def test_a_stale_rejection_again_is_a_duplicate(self) -> None: + v0 = self.baseline().state + late = self.request(v0, PATCH_SAFE_DIR, n=2) + self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=1)) + first = self.evaluate(late) + before = dump(self.path) + second = self.evaluate(late, clock=fixed_clock(999)) + self.assertIs(first.kind, ResultKind.STALE) + self.assertIs(second.kind, ResultKind.STALE) + self.assertTrue(second.duplicate) + self.assertEqual(dump(self.path), before) + + def test_the_switch_off_escalation_again_is_a_duplicate(self) -> None: + self.automatic_enabled = False + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + first = self.evaluate(request) + before = dump(self.path) + again = self.evaluate(request) + self.assertIs(first.kind, ResultKind.ESCALATED) + self.assertTrue(again.duplicate) + self.assertEqual(dump(self.path), before) + + def test_turning_the_switch_on_does_not_reopen_an_escalated_candidate(self) -> None: + """The stored decision is the decision: the candidate stays ESCALATED, no state appears.""" + self.automatic_enabled = False + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + self.evaluate(request) + again = self.evaluate(request, config=config_with(True)) + self.assertIs(again.kind, ResultKind.ESCALATED) + self.assertEqual(self.count("trusted_states"), 1) + + +class TestACrashAfterEachStage(IdempotencyCase): + def test_every_stage_boundary_is_recoverable(self) -> None: + for point in STAGES: + with self.subTest(after=point): + self.fresh_database() + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + with self.assertRaises(Stop): + self.evaluate(request, checkpoint=stop_after(point)) + self.assertEqual(check_database(self.path), []) + self.assertEqual(self.chain_problems(request, finished=False), []) + stored = self.read_candidate(request.candidate_id) + assert stored is not None + if point in ("after_decision", "after_request"): + self.assertIs(stored.status, CandidateStatus.PROMOTABLE) + self.assertEqual(self.current(), v0) + self.assertEqual(self.count("promotion_decisions"), 1) + else: + self.assertIn( + stored.status, + (CandidateStatus.BUILDING, CandidateStatus.ANALYZING), + ) + self.assertEqual(self.current(), v0) + self.assertEqual(self.count("trusted_states"), 1) + done = self.evaluate(request) + self.assertIs(done.kind, ResultKind.PROMOTED) + self.assertEqual(self.count("trusted_states"), 2) + self.assertEqual(self.chain_problems(request), []) + self.assertEqual(check_database(self.path), []) + + def test_a_crash_after_the_decision_leaves_a_promotable_candidate_the_rerun_commits( + self, + ) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + with self.assertRaises(Stop): + self.evaluate(request, checkpoint=stop_after("after_decision")) + stored = self.read_candidate(request.candidate_id) + assert stored is not None + self.assertIs(stored.status, CandidateStatus.PROMOTABLE) + done = self.evaluate(request) + assert done.state is not None + self.assertEqual(done.state.parent_state_id, v0.state_id) + self.assertEqual(self.count("trusted_states"), 2) + + def test_a_crash_after_a_non_promote_decision_is_a_duplicate_on_rerun(self) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_REGRESSION_DIR) + with self.assertRaises(Stop): + self.evaluate(request, checkpoint=stop_after("after_decision")) + before = dump(self.path) + again = self.evaluate(request) + self.assertIs(again.kind, ResultKind.REJECTED) + self.assertTrue(again.duplicate) + self.assertEqual(dump(self.path), before) + + def test_a_crash_while_the_candidate_is_stale_is_recoverable(self) -> None: + v0 = self.baseline().state + late = self.request(v0, PATCH_SAFE_DIR, n=2) + self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=1)) + with self.assertRaises(Stop): + self.evaluate(late, checkpoint=stop_after("after_verification")) + done = self.evaluate(late) + self.assertIs(done.kind, ResultKind.STALE) + self.assertEqual(self.count("trusted_states"), 2) + self.assertEqual(self.chain_problems(late), []) + + def read_candidate(self, candidate_id: str) -> Any: + with self.uow() as u: + return u.candidates.get(candidate_id) + + +class Counter: + """A database checkpoint that counts the commits of units of work and fails the n-th, once.""" + + def __init__(self, fail_at: int | None = None) -> None: + self.fail_at = fail_at + self.seen = 0 + + def __call__(self, name: str) -> None: + if name != "before_commit": + return + self.seen += 1 + if self.fail_at == self.seen: + raise InjectedFault(f"commit {self.seen}") + + +class TestAFailureAtEveryCommit(IdempotencyCase): + def test_a_failure_at_the_n_th_commit_is_recoverable_for_every_n(self) -> None: + self.fresh_database() + v0 = self.baseline().state + counter = Counter() + self.evaluate(self.request(v0, PATCH_SAFE_DIR), db_checkpoint=counter) + total = counter.seen + self.assertGreaterEqual(total, 10, "an evaluation opens many units of work") + for n in range(1, total + 1): + with self.subTest(commit=n, of=total): + self.fresh_database() + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + with self.assertRaises((InjectedFault, PersistenceError, PromotionCommitError)): + self.evaluate(request, db_checkpoint=Counter(fail_at=n)) + self.assertEqual(check_database(self.path), []) + self.assertEqual(self.chain_problems(request, finished=False), []) + done = self.evaluate(request) + self.assertIn(done.kind, (ResultKind.PROMOTED, ResultKind.DUPLICATE)) + self.assertEqual(self.count("trusted_states"), 2) + self.assertEqual(self.chain_problems(request), []) + self.assertEqual(check_database(self.path), []) + + +class TestAFailureInsideThePromotion(IdempotencyCase): + POINTS = ( + "after_trusted_state_insert", + "after_invariant_refs_insert", + "after_lineage_head_update", + "after_decision_commit", + ) + + def test_the_promotion_is_atomic_and_the_same_request_then_commits(self) -> None: + for point in self.POINTS: + with self.subTest(point=point): + self.fresh_database() + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + gate = Gate(point, fail=True) + gate.release.set() + with self.assertRaises(PromotionCommitError): + self.evaluate(request, db_checkpoint=gate) + self.assertTrue(gate.fired) + self.assertEqual(self.current(), v0) + self.assertEqual(self.count("trusted_states"), 1) + stored = self.read_candidate_status(request.candidate_id) + self.assertEqual(stored, CandidateStatus.PROMOTABLE.value) + self.assertEqual(check_database(self.path), []) + done = self.evaluate(request) + self.assertIs(done.kind, ResultKind.PROMOTED) + self.assertEqual(self.count("trusted_states"), 2) + self.assertEqual(self.chain_problems(request, finished=True), []) + + def read_candidate_status(self, candidate_id: str) -> str: + return str( + self.scalar("SELECT status FROM candidates WHERE candidate_id = ?", (candidate_id,)) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_e2e_lifecycles.py b/tests/unit/test_e2e_lifecycles.py new file mode 100644 index 0000000..d401a70 --- /dev/null +++ b/tests/unit/test_e2e_lifecycles.py @@ -0,0 +1,206 @@ +"""The canonical lifecycle through the database (P6c step 5; Doc 12 §14, Doc 06 §25-§26, Doc 09 §33, +§34, Doc 15 §13.3 and §23 M3 inputs; C-22, C-45, C-164, C-170, C-173). + +HAND EXPECTATIONS (CLAUDE.md, the fixtures' own ``expected.json`` and the P6b prompt): + +* Doc 12 §14, the eleven steps: 1 create the initial Trusted State (version 0, public SSH open: + INV-SEC-001 VIOLATED, the other three PROTECTED); 2 generate a candidate from the current parent; + 3 analyze it; 4 verify the affected invariants (at least the three the fixture names as + ``required_affected``: INV-FUNC-001, INV-SEC-001, INV-SEC-002); 5 reject the unsafe candidate + (INV-FUNC-001 is the only failing required check); 6 confirm Trusted State is unchanged; + 7 generate the safe candidate; 8 promote it atomically; 9 confirm the new Trusted State has the + parent, the version and the evidence: version 1, parent version 0, INV-SEC-001 VERIFIED (a + VIOLATED reference cannot be carried), INV-SEC-003 CARRIED_FORWARD (unaffected, eligible + evidence), every reference carrying the binding record bound to version 1; 10 attempt to promote + a stale candidate (written against version 0 after version 1 exists); 11 confirm it cannot + advance the state: REJECT STALE_PARENT, version 1 still current, two states in all; +* a third candidate, written against version 1, carries what it leaves unaffected forward through + version 1's binding record and is promoted into version 2. +""" + +from __future__ import annotations + +import unittest +from typing import Any + +from core.application.chain import describe_chain +from core.application.orchestrator import EvaluationResult, ResultKind +from core.domain.enums import CandidateStatus, DecisionType, InvariantStatus, ProofOrigin +from core.domain.evidence import EvidenceContext, EvidenceKind +from core.domain.promotion import DecisionReason, ProofSource +from core.promotion import ids as pids +from evidence.service import EvidenceService +from tests.promotion_commit_builders import check_database +from tests.promotion_orchestrator_builders import ( + FIXTURES, + PATCH_REGRESSION_DIR, + PATCH_SAFE_DIR, + OrchestratorCase, +) + +SEC1, SEC2, SEC3, FUNC = "INV-SEC-001", "INV-SEC-002", "INV-SEC-003", "INV-FUNC-001" +NEXT_DIR = FIXTURES / "verification" / "ec2_rds_5432_pass_v1" +REQUIRED_AFFECTED = {FUNC, SEC1, SEC2} # fixtures/canonical_regression/*/expected.json + + +class TestTheTwelveStepLifecycle(OrchestratorCase): + def chain_problems(self, result: EvaluationResult) -> list[str]: + with self.uow() as u: + return describe_chain(u, result.candidate.candidate_id).problems() + + def test_doc_12_14_steps_1_to_11(self) -> None: + # 1 create the initial Trusted State + baseline = self.baseline() + v0 = baseline.state + self.assertEqual(v0.version, 0) + self.assertEqual( + {r.invariant_id: r.status for r in v0.invariant_refs}, + { + SEC1: InvariantStatus.VIOLATED, + SEC2: InvariantStatus.PROTECTED, + SEC3: InvariantStatus.PROTECTED, + FUNC: InvariantStatus.PROTECTED, + }, + ) + + # 2 generate a candidate from the current parent, 3 analyze, 4 verify, 5 reject + unsafe = self.evaluate(self.request(v0, PATCH_REGRESSION_DIR, n=1)) + self.assertEqual(unsafe.candidate.parent_state_id, v0.state_id) + assert unsafe.impact is not None and unsafe.verification is not None + self.assertEqual(self.chain_problems(unsafe), []) + self.assertIn( + "IMPACT", [e.kind.value for e in self.records(unsafe) if e.kind is EvidenceKind.IMPACT] + ) + verified = { + r.invariant_id for r in unsafe.verification.results if r.invariant_id is not None + } + self.assertTrue(verified >= REQUIRED_AFFECTED, verified) + self.assertIs(unsafe.kind, ResultKind.REJECTED) + assert unsafe.decision is not None + self.assertIs(unsafe.decision.decision, DecisionType.REJECT) + failing = { + c.target + for c in unsafe.decision.blocking_conditions + if c.code is DecisionReason.REQUIRED_CHECK_FAILED + } + self.assertEqual(failing, {f"INVARIANT:{FUNC}:1"}) + self.assertIs(unsafe.candidate.status, CandidateStatus.REJECTED) + + # 6 confirm Trusted State is unchanged + self.assertEqual(self.current(), v0) + self.assertEqual(self.count("trusted_states"), 1) + self.assertEqual(self.count("lineage_heads"), 1) + self.assertEqual(self.read_state(v0.state_id), v0) + + # 7 generate the safe candidate and, in parallel, a second one that will become stale + stale_request = self.request(v0, PATCH_SAFE_DIR, n=3) + + # 8 promote atomically + safe = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=2)) + self.assertIs(safe.kind, ResultKind.PROMOTED) + assert safe.state is not None and safe.decision is not None and safe.outcome is not None + self.assertEqual(self.chain_problems(safe), []) + self.assertEqual(self.count("trusted_states"), 2) + self.assertEqual(self.current(), safe.state) + + # 9 the new Trusted State: parent, version, evidence + v1 = safe.state + self.assertEqual((v1.version, v1.parent_state_id), (1, v0.state_id)) + self.assertEqual(v1.commit_decision_id, safe.decision.decision_id) + by = {r.invariant_id: r for r in v1.invariant_refs} + self.assertEqual({r.status for r in v1.invariant_refs}, {InvariantStatus.PROTECTED}) + self.assertIs(by[SEC1].origin, ProofOrigin.VERIFIED) + self.assertIs(by[SEC3].origin, ProofOrigin.CARRIED_FORWARD) + plan = {e.invariant_id: e.source for e in safe.outcome.proof_plan} + self.assertIs(plan[SEC1], ProofSource.VERIFIED) + self.assertIs(plan[SEC3], ProofSource.CARRIED_FORWARD) + self.assertEqual( + {i: (s is ProofSource.VERIFIED) for i, s in plan.items()}, + {i: by[i].origin is ProofOrigin.VERIFIED for i in plan}, + ) + binding_id = pids.evidence_id_for(safe.decision.decision_id) + for invariant_id, ref in by.items(): + self.assertIn(binding_id, ref.evidence_ids, invariant_id) + self.assertIn(binding_id, v1.evidence_refs) + service = EvidenceService() + with self.uow() as u: + binding = u.evidence.get(binding_id) + assert binding is not None + self.assertIs(binding.kind, EvidenceKind.STATE) + self.assertEqual((binding.state_id, binding.state_hash), (v1.state_id, v1.state_hash)) + self.assertTrue( + service.usable_as_proof(u, binding_id, EvidenceContext.state(v1.state_id)).usable + ) + # the sources' evidence is proof for their own context, not for version 1 (C-52, C-164) + carried = [e for e in by[SEC3].evidence_ids if e != binding_id] + self.assertTrue(carried) + for evidence_id in carried: + self.assertFalse( + service.usable_as_proof( + u, evidence_id, EvidenceContext.state(v1.state_id) + ).usable + ) + self.assertTrue( + service.usable_as_proof( + u, evidence_id, EvidenceContext.state(v0.state_id) + ).usable + ) + + # 10 attempt to promote a stale candidate, 11 confirm it cannot advance the state + stale = self.evaluate(stale_request) + self.assertIs(stale.kind, ResultKind.STALE) + assert stale.decision is not None + self.assertIs(stale.decision.decision, DecisionType.REJECT) + self.assertIs(stale.decision.reason, DecisionReason.STALE_PARENT) + self.assertIs(stale.candidate.status, CandidateStatus.REJECTED) + self.assertEqual(self.current(), v1) + self.assertEqual(self.count("trusted_states"), 2) + self.assertEqual(self.chain_problems(stale), []) + self.assertEqual(check_database(self.path), []) + + def records(self, result: EvaluationResult) -> list[Any]: + with self.uow() as u: + return list(u.evidence.list_for_candidate(result.candidate.candidate_id)) + + def read_state(self, state_id: str) -> Any: + with self.uow() as u: + return u.trusted_states.get(state_id) + + +class TestAThirdCandidateCarriesThroughVersionOnesBinding(OrchestratorCase): + def test_a_candidate_from_version_one_is_promoted_into_version_two(self) -> None: + v0 = self.baseline().state + first = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=1)) + assert first.state is not None + v1 = first.state + second = self.evaluate(self.request(v1, NEXT_DIR, n=2)) + self.assertIs(second.kind, ResultKind.PROMOTED) + assert ( + second.state is not None and second.outcome is not None and second.decision is not None + ) + v2 = second.state + self.assertEqual((v2.version, v2.parent_state_id), (2, v1.state_id)) + self.assertEqual(self.current(), v2) + carried = [ + e.invariant_id + for e in second.outcome.proof_plan + if e.source is ProofSource.CARRIED_FORWARD + ] + self.assertTrue(carried, "the third candidate leaves something unaffected") + v1_binding = pids.evidence_id_for(v1.commit_decision_id or "") + v2_binding = pids.evidence_id_for(second.decision.decision_id) + by = {r.invariant_id: r for r in v2.invariant_refs} + for invariant_id in carried: + with self.subTest(invariant=invariant_id): + self.assertIn(v1_binding, by[invariant_id].evidence_ids) + self.assertIn(v2_binding, by[invariant_id].evidence_ids) + for invariant_id, ref in by.items(): + self.assertIn(v2_binding, ref.evidence_ids, invariant_id) + with self.uow() as u: + for result in (first, second): + self.assertEqual(describe_chain(u, result.candidate.candidate_id).problems(), []) + self.assertEqual(check_database(self.path), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_e2e_properties.py b/tests/unit/test_e2e_properties.py new file mode 100644 index 0000000..9fb5f91 --- /dev/null +++ b/tests/unit/test_e2e_properties.py @@ -0,0 +1,233 @@ +"""Seeded properties of ``evaluate`` over random sequences of candidates (P6c step 5; Doc 09 §26, +§39; Doc 15 §13.3; stdlib ``random`` only). + +For each seed a lineage starts from the canonical baseline and then 1 to 4 candidates are evaluated +in turn; each is a randomly chosen fixture configuration (the 18 fixtures with a parent), written +against the state that is current when it is evaluated, with a random attempt number 1 to 4. + +PROPERTIES, each from the P6c prompt and Doc 15 §13.3 ("no unsafe candidate can advance Trusted +State, and a valid candidate can advance exactly once"): + +* **lineage**: the trusted states are a chain with no gap (versions 0 to k, each the child of the + previous, one head); +* **non-promotion**: an evaluation that does not end in a promotion changes no row of + ``trusted_states`` and no lineage head; a promotion appends exactly one; +* **chains**: the chain of every evaluated candidate is complete (``problems() == []``); +* **replay**: evaluating the same requests again, in order, by another orchestrator, writes nothing; +* **switch**: with ``promotion.automatic_enabled`` false no candidate is ever promoted. + +``TestTheCheckersFire`` shows that each property's checker fails on a database or a pair of dumps +that breaks it, so a checker that can no longer fail is itself caught. +""" + +from __future__ import annotations + +import random +import sqlite3 +import unittest +from contextlib import closing +from pathlib import Path +from typing import Any + +from core.application.chain import describe_chain +from core.application.orchestrator import EvaluationRequest, EvaluationResult, ResultKind +from tests.pipeline_builders import fixed_clock +from tests.promotion_commit_builders import check_database, dump +from tests.promotion_orchestrator_builders import ( + PATCH_SAFE_DIR, + OrchestratorCase, +) +from tests.promotion_pipeline import ROOT, fixtures_with_a_parent + +SEEDS = (11, 12, 13, 14, 15, 16) +SWITCH_OFF_SEEDS = (21, 22, 23) + + +def lineage_problems(path: str | Path) -> list[str]: + """The trusted states form a chain with no gap and one head per lineage.""" + problems: list[str] = [] + with closing(sqlite3.connect(path)) as conn: + rows = conn.execute( + "SELECT lineage_id, version, state_id, parent_state_id FROM trusted_states " + "ORDER BY lineage_id, version" + ).fetchall() + heads = dict(conn.execute("SELECT lineage_id, current_state_id FROM lineage_heads")) + by_lineage: dict[str, list[tuple[Any, ...]]] = {} + for row in rows: + by_lineage.setdefault(row[0], []).append(row) + for lineage, states in by_lineage.items(): + if [s[1] for s in states] != list(range(len(states))): + problems.append(f"{lineage}: versions {[s[1] for s in states]} have a gap") + for before, after in zip(states, states[1:], strict=False): + if after[3] != before[2]: + problems.append(f"{lineage}: version {after[1]} is not the child of {before[1]}") + if heads.get(lineage) != states[-1][2]: + problems.append(f"{lineage}: the head is not the newest state") + return problems + + +def trusted_rows(path: str | Path) -> list[tuple[Any, ...]]: + with closing(sqlite3.connect(path)) as conn: + return [ + tuple(r) for r in conn.execute("SELECT * FROM trusted_states ORDER BY rowid").fetchall() + ] + [tuple(r) for r in conn.execute("SELECT * FROM lineage_heads ORDER BY rowid")] + + +def promotion_delta_problems( + before: list[tuple[Any, ...]], after: list[tuple[Any, ...]], promoted: bool +) -> list[str]: + """A non-promotion changes nothing; a promotion adds one state and moves the head.""" + if not promoted: + return [] if before == after else ["trusted state changed without a promotion"] + new_states = [row for row in after if row not in before] + gone = [row for row in before if row not in after] + # one new state row, and the head row replaced by one that names it + heads_moved = len(gone) == 1 and len(new_states) == 2 + return [] if heads_moved else [f"a promotion changed {len(new_states)} rows, lost {len(gone)}"] + + +class PropertyCase(OrchestratorCase): + def sequence(self, seed: int) -> list[tuple[EvaluationRequest, EvaluationResult]]: + """Run one seeded sequence, checking the per-step properties as it goes.""" + rng = random.Random(seed) + self.fresh_database() + self.baseline() + fixtures = list(fixtures_with_a_parent().values()) + done: list[tuple[EvaluationRequest, EvaluationResult]] = [] + for step in range(rng.randint(1, 4)): + fixture = rng.choice(fixtures) + attempt = rng.randint(1, 4) + request = self.request( + self.current(), ROOT / fixture.data["config"], n=step + 1, attempt=attempt + ) + before = trusted_rows(self.path) + result = self.evaluate(request) + promoted = result.kind is ResultKind.PROMOTED + self.assertEqual( + promotion_delta_problems(before, trusted_rows(self.path), promoted), + [], + f"seed {seed}, step {step}, {fixture.data['fixture_id']}: {result.kind}", + ) + done.append((request, result)) + return done + + def chains_are_complete(self, done: list[tuple[EvaluationRequest, EvaluationResult]]) -> None: + with self.uow() as u: + for request, _ in done: + self.assertEqual(describe_chain(u, request.candidate_id).problems(), []) + + +class TestRandomSequences(PropertyCase): + def test_the_properties_hold_for_every_seed(self) -> None: + kinds: set[ResultKind] = set() + promotions = 0 + for seed in SEEDS: + with self.subTest(seed=seed): + done = self.sequence(seed) + self.assertEqual(lineage_problems(self.path), []) + self.assertEqual(check_database(self.path), []) + self.chains_are_complete(done) + promotions += sum(1 for _, r in done if r.kind is ResultKind.PROMOTED) + self.assertEqual(self.count("trusted_states"), 1 + promotions_in(done)) + kinds |= {r.kind for _, r in done} + # the seeds reach more than one kind of ending and at least one promotion + self.assertGreater(len(kinds), 1, kinds) + self.assertGreater(promotions, 0) + + def test_replaying_the_requests_changes_nothing(self) -> None: + for seed in SEEDS: + with self.subTest(seed=seed): + done = self.sequence(seed) + before = dump(self.path) + for request, first in done: + again = self.evaluate(request, clock=fixed_clock(900)) + self.assertTrue(again.duplicate, (request.candidate_id, first.kind)) + self.assertEqual(dump(self.path), before) + + def test_a_promotion_happens_at_most_once_per_candidate(self) -> None: + for seed in SEEDS: + with self.subTest(seed=seed): + done = self.sequence(seed) + self.assertEqual( + self.count("promotion_decisions", "committed_state_id IS NOT NULL"), + promotions_in(done), + ) + + +class TestWithTheSwitchOff(PropertyCase): + automatic_enabled = False + + def test_nothing_is_ever_promoted(self) -> None: + for seed in SWITCH_OFF_SEEDS: + with self.subTest(seed=seed): + done = self.sequence(seed) + self.assertEqual(self.count("trusted_states"), 1) + self.assertNotIn(ResultKind.PROMOTED, {r.kind for _, r in done}) + self.assertEqual(lineage_problems(self.path), []) + self.chains_are_complete(done) + + +def promotions_in(done: list[tuple[EvaluationRequest, EvaluationResult]]) -> int: + return sum(1 for _, result in done if result.kind is ResultKind.PROMOTED) + + +class TestTheCheckersFire(OrchestratorCase): + """Each property's checker, shown to fail where its rule is broken.""" + + def promoted_database(self) -> None: + v0 = self.baseline().state + self.evaluate(self.request(v0, PATCH_SAFE_DIR)) + + def test_the_lineage_checker_finds_a_gap(self) -> None: + self.promoted_database() + self.assertEqual(lineage_problems(self.path), []) + self.tamper("UPDATE trusted_states SET version = 2 WHERE version = 1") + self.assertTrue(any("gap" in p for p in lineage_problems(self.path))) + + def test_the_lineage_checker_finds_a_broken_parent_link(self) -> None: + self.promoted_database() + self.tamper("UPDATE trusted_states SET parent_state_id = state_id WHERE version = 1") + self.assertTrue(any("child" in p for p in lineage_problems(self.path))) + + def test_the_lineage_checker_finds_a_head_that_is_not_the_newest_state(self) -> None: + self.promoted_database() + self.tamper( + "UPDATE lineage_heads SET current_state_id = " + "(SELECT state_id FROM trusted_states WHERE version = 0)" + ) + self.assertTrue(any("head" in p for p in lineage_problems(self.path))) + + def test_the_delta_checker_finds_a_change_without_a_promotion(self) -> None: + self.promoted_database() + after = trusted_rows(self.path) + self.assertEqual(promotion_delta_problems(after, after, False), []) + shorter = after[:1] + self.assertTrue(promotion_delta_problems(shorter, after, False)) + + def test_the_delta_checker_finds_a_promotion_that_changed_nothing_or_too_much(self) -> None: + self.promoted_database() + after = trusted_rows(self.path) + self.assertTrue(promotion_delta_problems(after, after, True)) + self.assertEqual(promotion_delta_problems(after[:1] + after[2:], after, True) != [], True) + + def test_the_chain_checker_finds_a_deleted_event(self) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + self.evaluate(request) + with self.uow() as u: + self.assertEqual(describe_chain(u, request.candidate_id).problems(), []) + self.tamper("DELETE FROM audit_events WHERE event_type = 'IMPACT_COMPLETED'") + with self.uow() as u: + self.assertTrue(describe_chain(u, request.candidate_id).problems()) + + def test_the_replay_check_finds_a_write(self) -> None: + self.promoted_database() + before = dump(self.path) + self.tamper( + "INSERT INTO lineage_heads VALUES ('99999999-9999-9999-9999-999999999999', 'x')" + ) + self.assertNotEqual(dump(self.path), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_orchestrator.py b/tests/unit/test_orchestrator.py new file mode 100644 index 0000000..4b357a2 --- /dev/null +++ b/tests/unit/test_orchestrator.py @@ -0,0 +1,612 @@ +"""``EvaluationOrchestrator.evaluate`` (P6c step 4; Doc 03 §15; Doc 06 §20, §27; Doc 09 §20, §25, +§26, §40; Doc 11 §43; C-170, C-172, C-173, C-174). + +HAND EXPECTATIONS (CLAUDE.md, the prompt and the fixtures' own ``expected.json``): + +* the stages run in the order of the objective: normalize, identity, graphs, impact, scope policy, + verify, decide, record the decision, commit only a PROMOTE; +* ``patch_safe`` from the canonical baseline is PROMOTED into version 1; ``patch_regression`` is + REJECTED (INV-FUNC-001 is the only failing check) and version 0 stays current; every decision is + stored; +* with ``promotion.automatic_enabled`` false a would-be PROMOTE is an audited ESCALATE with + ``AUTOMATIC_PROMOTION_DISABLED`` and no state change; the orchestrator refuses to start when the + flag is not a bool; +* a directory that does not normalize leaves a FAILED candidate, an ERROR record, no decision and + the state unchanged; a verifier crash is a VERIFIER_ERROR result and an ESCALATE; +* a candidate whose parent is no longer current is REJECTED ``STALE_PARENT`` and the state stays; +* the same request again is a duplicate (nothing written); the same ids with another directory + raise ``ConflictingDuplicateError``; +* the orchestrator's source names no verification result, invariant status or PASS/FAIL/UNKNOWN. +""" + +from __future__ import annotations + +import ast +import tempfile +import unittest +from pathlib import Path +from typing import Any +from unittest import mock + +from core.application import ids as oids +from core.application.chain import describe_chain +from core.application.orchestrator import EvaluationOrchestrator, ResultKind +from core.application.scope_policy import DailScopePolicy +from core.domain.audit import AuditEventType +from core.domain.enums import CandidateStatus, DecisionType, VerificationResult +from core.domain.errors import DomainValidationError +from core.domain.evidence import EvidenceKind +from core.domain.promotion import ConflictingDuplicateError, DecisionReason +from core.promotion import ids as pids +from core.verification.dispatch import DEFAULT_VERIFIERS, REACHABILITY_CFG +from core.verification.outcome import PredicateOutcome +from tests.domain_builders import uid +from tests.pipeline_builders import fixed_clock +from tests.promotion_commit_builders import InjectedFault, dump +from tests.promotion_orchestrator_builders import ( + PATCH_REGRESSION_DIR, + PATCH_SAFE_DIR, + OrchestratorCase, + config_with, +) + +REPO = Path(__file__).resolve().parents[2] +SEC1, SEC3, FUNC = "INV-SEC-001", "INV-SEC-003", "INV-FUNC-001" + + +def raising(_definition: Any, _inputs: Any) -> Any: + raise RuntimeError("the reachability verifier crashed") + + +def unknown(_definition: Any, _inputs: Any) -> Any: + from core.domain.verification import VerificationReason + + return PredicateOutcome( + VerificationResult.UNKNOWN, VerificationReason.RULE_INDETERMINATE, None, {}, () + ) + + +class TestTheSourceCoordinatesOnly(unittest.TestCase): + """C-170 (1): no decision logic. The source is read with ``ast`` and never imported.""" + + FORBIDDEN = {"VerificationResult", "InvariantStatus", "PASS", "FAIL", "UNKNOWN"} + + def names(self) -> set[str]: + tree = ast.parse((REPO / "core" / "application" / "orchestrator.py").read_text("utf-8")) + found: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Name): + found.add(node.id) + elif isinstance(node, ast.Attribute): + found.add(node.attr) + elif isinstance(node, ast.alias): + found.add(node.name.split(".")[-1]) + if node.asname: + found.add(node.asname) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + found.add(node.value) + return found + + def test_it_names_none_of_the_result_vocabulary(self) -> None: + self.assertEqual(self.names() & self.FORBIDDEN, set()) + + def test_the_check_can_fail(self) -> None: + """The scan finds each forbidden word in a source that uses it.""" + for source in ( + "from core.domain.enums import VerificationResult\n", + "if r.status is InvariantStatus.PROTECTED: pass\n", + "x = PASS\n", + "if r.result == 'FAIL': pass\n", + "y = a.UNKNOWN\n", + ): + with self.subTest(source=source): + tree = ast.parse(source) + found = set() + for node in ast.walk(tree): + if isinstance(node, ast.Name): + found.add(node.id) + elif isinstance(node, ast.Attribute): + found.add(node.attr) + elif isinstance(node, ast.alias): + found.add(node.name) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + found.add(node.value) + self.assertTrue(found & self.FORBIDDEN, source) + + def test_it_has_no_if_on_an_invariant_id(self) -> None: + tree = ast.parse((REPO / "core" / "application" / "orchestrator.py").read_text("utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.If | ast.IfExp | ast.While): + tested = {n.attr for n in ast.walk(node.test) if isinstance(n, ast.Attribute)} | { + n.id for n in ast.walk(node.test) if isinstance(n, ast.Name) + } + self.assertEqual( + tested & {"invariant_id", "invariant_version", "result", "reason"}, + set(), + f"line {node.lineno}", + ) + + +class TestConstruction(OrchestratorCase): + def test_a_flag_that_is_not_a_bool_stops_startup(self) -> None: + class Config: + def get(self, key: str) -> Any: + return "yes" if key == "promotion.automatic_enabled" else None + + with self.assertRaises(DomainValidationError): + self.orchestrator(config=Config()) # type: ignore[arg-type] + + def test_a_missing_flag_stops_startup(self) -> None: + class Config: + def get(self, key: str) -> Any: + raise KeyError(key) + + with self.assertRaises(DomainValidationError): + self.orchestrator(config=Config()) # type: ignore[arg-type] + + def test_the_real_configuration_starts_it_with_the_flag_off(self) -> None: + self.assertIsInstance(self.orchestrator(config=config_with(False)), EvaluationOrchestrator) + + def test_other_arguments_are_checked(self) -> None: + for bad in ( + {"scope_policy": object()}, + {"clock": "now"}, + {"policy_version": ""}, + {"checkpoint": 3}, + ): + with self.subTest(bad=bad), self.assertRaises(DomainValidationError): + self.orchestrator(**bad) + with self.assertRaises(DomainValidationError): + EvaluationOrchestrator( + "not a factory", # type: ignore[arg-type] + config=config_with(True), + scope_policy=DailScopePolicy(), + clock=fixed_clock(60), + policy_version="p", + ) + + +class TestTheRequest(OrchestratorCase): + def test_bad_requests_are_refused_before_anything_is_written(self) -> None: + v0 = self.baseline().state + before = dump(self.path) + good = self.request(v0, PATCH_SAFE_DIR) + import dataclasses + + for bad in ( + {"candidate_id": "x"}, + {"lineage_id": "x"}, + {"attempt_number": 0}, + {"attempt_number": True}, + {"candidate_sequence": 0}, + {"candidate_dir": "fixtures"}, + {"definitions": ()}, + {"bindings": [1]}, + {"context": object()}, + {"patch": "diff"}, + ): + with self.subTest(bad=bad), self.assertRaises(DomainValidationError): + dataclasses.replace(good, **bad) + self.assertEqual(dump(self.path), before) + + def test_only_a_fixed_patch_is_accepted(self) -> None: + import dataclasses + + from core.domain.enums import CandidateSource + from core.domain.patch import Patch + + v0 = self.baseline().state + llm = Patch.create( + source=CandidateSource.LLM, + content="x\n", + parent_state_id=v0.state_id, + now=v0.created_at, + llm_request_id=uid(1), + ) + with self.assertRaises(DomainValidationError): + dataclasses.replace(self.request(v0, PATCH_SAFE_DIR), patch=llm) + + def test_a_request_without_a_baseline_is_refused(self) -> None: + import dataclasses + + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + other = dataclasses.replace(request, lineage_id=uid(999)) + with self.assertRaises(DomainValidationError): + self.evaluate(other) + + def test_a_patch_for_a_state_that_is_not_stored_is_refused(self) -> None: + import dataclasses + + from core.domain.enums import CandidateSource + from core.domain.patch import Patch + + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + patch = Patch.create( + source=CandidateSource.FIXED_PATCH, + content="x\n", + parent_state_id=uid(98765), + now=v0.created_at, + ) + with self.assertRaises(DomainValidationError): + self.evaluate(dataclasses.replace(request, patch=patch)) + + +class TestTheStageOrder(OrchestratorCase): + def test_the_stages_run_in_the_order_of_the_objective(self) -> None: + import core.application.orchestrator as module + + v0 = self.baseline().state + calls: list[str] = [] + + def spy(name: str) -> Any: + real = getattr(module, name) + + def wrapper(*args: Any, **kwargs: Any) -> Any: + calls.append(name) + return real(*args, **kwargs) + + return wrapper + + class Scope(DailScopePolicy): + def select(self, *args: Any, **kwargs: Any) -> Any: + calls.append("select") + return super().select(*args, **kwargs) + + names = ( + "normalize_directory", + "compare", + "build_graph", + "compute_impact", + "run_work_set", + "decide", + "record_decision", + "commit_promotion", + ) + patches = [mock.patch.object(module, n, spy(n)) for n in names] + for p in patches: + p.start() + self.addCleanup(p.stop) + self.evaluate(self.request(v0, PATCH_SAFE_DIR), scope_policy=Scope()) + self.assertEqual( + calls, + [ + "normalize_directory", + "compare", + "build_graph", # the parent's graph + "build_graph", # the candidate's graph + "compute_impact", + "select", + "run_work_set", + "decide", + "record_decision", + "commit_promotion", + ], + ) + + def test_a_non_promote_never_reaches_the_commit(self) -> None: + import core.application.orchestrator as module + + v0 = self.baseline().state + with mock.patch.object(module, "commit_promotion") as commit: + result = self.evaluate(self.request(v0, PATCH_REGRESSION_DIR)) + self.assertIs(result.kind, ResultKind.REJECTED) + commit.assert_not_called() + + +class TestTheCanonicalDecisions(OrchestratorCase): + def test_the_safe_repair_is_promoted_into_version_one(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_SAFE_DIR)) + self.assertIs(result.kind, ResultKind.PROMOTED) + assert result.state is not None and result.decision is not None + self.assertEqual(result.state.version, 1) + self.assertEqual(result.state.parent_state_id, v0.state_id) + self.assertEqual(self.current(), result.state) + self.assertIs(result.decision.decision, DecisionType.PROMOTE) + self.assertEqual(result.decision.committed_state_id, result.state.state_id) + self.assertIs(result.candidate.status, CandidateStatus.PROMOTED) + self.assertFalse(result.duplicate) + + def test_the_regression_is_rejected_and_version_zero_stays(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_REGRESSION_DIR)) + self.assertIs(result.kind, ResultKind.REJECTED) + assert result.decision is not None + self.assertIs(result.decision.decision, DecisionType.REJECT) + self.assertEqual(self.current(), v0) + self.assertEqual(self.count("trusted_states"), 1) + self.assertEqual( + {c.target for c in result.decision.blocking_conditions if c.target.startswith("INV")}, + {f"INVARIANT:{FUNC}:1"}, + ) + self.assertIs(result.candidate.status, CandidateStatus.REJECTED) + + def test_every_decision_is_stored(self) -> None: + v0 = self.baseline().state + rejected = self.evaluate(self.request(v0, PATCH_REGRESSION_DIR, n=1)) + promoted = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=2)) + with self.uow() as u: + for result in (rejected, promoted): + assert result.decision is not None + self.assertEqual(u.decisions.get(result.decision.decision_id), result.decision) + self.assertEqual( + rejected.decision.decision_id if rejected.decision else None, + oids.decision_id_for(uid(3401), 1), + ) + + def test_the_chains_are_complete(self) -> None: + v0 = self.baseline().state + a = self.evaluate(self.request(v0, PATCH_REGRESSION_DIR, n=1)) + b = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=2)) + with self.uow() as u: + for result in (a, b): + self.assertEqual(describe_chain(u, result.candidate.candidate_id).problems(), []) + + def test_the_result_carries_the_reports_for_the_caller(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_SAFE_DIR)) + assert result.impact is not None and result.verification is not None + self.assertEqual(result.impact.candidate_id, result.candidate.candidate_id) + self.assertEqual(result.verification.candidate_id, result.candidate.candidate_id) + self.assertEqual( + set(result.evidence_ids), {r.verification_id for r in result.verification.results} + ) + + +class TestTheEmergencySwitch(OrchestratorCase): + automatic_enabled = False + + def test_a_would_be_promote_is_an_audited_escalate(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_SAFE_DIR)) + self.assertIs(result.kind, ResultKind.ESCALATED) + assert result.decision is not None + self.assertIs(result.decision.decision, DecisionType.ESCALATE) + self.assertIs(result.decision.reason, DecisionReason.AUTOMATIC_PROMOTION_DISABLED) + self.assertEqual(self.current(), v0) + self.assertEqual(self.count("trusted_states"), 1) + self.assertIs(result.candidate.status, CandidateStatus.ESCALATED) + with self.uow() as u: + types = [ + e.event_type for e in u.audit.list_for_candidate(result.candidate.candidate_id) + ] + self.assertIn(AuditEventType.ESCALATED, types) + self.assertNotIn(AuditEventType.STATE_PROMOTED, types) + self.assertNotIn(AuditEventType.PROMOTION_REQUESTED, types) + + +class TestVerifierFailures(OrchestratorCase): + def test_a_crashing_verifier_is_a_verifier_error_and_an_escalate(self) -> None: + v0 = self.baseline().state + table = {**DEFAULT_VERIFIERS, REACHABILITY_CFG: raising} + result = self.evaluate(self.request(v0, PATCH_SAFE_DIR), verifiers=table) + self.assertIs(result.kind, ResultKind.ESCALATED) + assert result.decision is not None and result.verification is not None + self.assertIs(result.decision.reason, DecisionReason.VERIFIER_ERROR) + self.assertIn( + VerificationResult.VERIFIER_ERROR, {r.result for r in result.verification.results} + ) + self.assertEqual(self.current(), v0) + + def test_an_unknown_result_is_a_retry_and_the_budget_ends_in_an_escalate(self) -> None: + v0 = self.baseline().state + table = {**DEFAULT_VERIFIERS, REACHABILITY_CFG: unknown} + first = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=1, attempt=1), verifiers=table) + self.assertIs(first.kind, ResultKind.RETRY_REQUIRED) + self.assertIs(first.candidate.status, CandidateStatus.RETRY_REQUIRED) + with self.uow() as u: + types = [e.event_type for e in u.audit.list_for_candidate(first.candidate.candidate_id)] + self.assertEqual(types[-1], AuditEventType.RETRY_SCHEDULED) + last = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=2, attempt=3), verifiers=table) + self.assertIs(last.kind, ResultKind.ESCALATED) + assert last.decision is not None + self.assertIs(last.decision.reason, DecisionReason.RETRY_BUDGET_EXHAUSTED) + self.assertEqual(self.current(), v0) + + +class TestConstructionFailure(OrchestratorCase): + def broken_dir(self) -> Path: + tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + (tmp / "main.tf").write_text('resource "aws_instance" {{{\n', encoding="utf-8") + return tmp + + def test_the_candidate_fails_with_an_error_record_and_no_decision(self) -> None: + v0 = self.baseline().state + request = self.request(v0, self.broken_dir()) + result = self.evaluate(request) + self.assertIs(result.kind, ResultKind.CONSTRUCTION_FAILED) + self.assertIs(result.candidate.status, CandidateStatus.FAILED) + self.assertIsNone(result.decision) + self.assertIsNone(result.state) + self.assertEqual(self.count("promotion_decisions"), 0) + self.assertEqual(self.current(), v0) + with self.uow() as u: + errors = [ + e + for e in u.evidence.list_for_correlation(request.context.correlation_id) + if e.kind is EvidenceKind.ERROR + ] + types = [e.event_type for e in u.audit.list_for_candidate(request.candidate_id)] + self.assertEqual(len(errors), 1) + self.assertEqual(types, [AuditEventType.CANDIDATE_CREATED, AuditEventType.ERROR_OCCURRED]) + with self.uow() as u: + self.assertEqual(describe_chain(u, request.candidate_id).problems(), []) + + def test_a_rerun_is_a_duplicate(self) -> None: + v0 = self.baseline().state + request = self.request(v0, self.broken_dir()) + first = self.evaluate(request) + before = dump(self.path) + second = self.evaluate(request) + self.assertIs(second.kind, ResultKind.CONSTRUCTION_FAILED) + self.assertTrue(second.duplicate) + self.assertEqual(second.candidate, first.candidate) + self.assertEqual(dump(self.path), before) + + +class TestTheStaleRoute(OrchestratorCase): + def test_a_candidate_written_against_v0_is_stale_once_v1_exists(self) -> None: + v0 = self.baseline().state + stale_request = self.request(v0, PATCH_SAFE_DIR, n=2) + winner = self.evaluate(self.request(v0, PATCH_SAFE_DIR, n=1)) + self.assertIs(winner.kind, ResultKind.PROMOTED) + result = self.evaluate(stale_request) + self.assertIs(result.kind, ResultKind.STALE) + assert result.decision is not None + self.assertIs(result.decision.decision, DecisionType.REJECT) + self.assertIs(result.decision.reason, DecisionReason.STALE_PARENT) + self.assertEqual( + result.decision.decision_id, + pids.rejection_decision_id_for(oids.decision_id_for(stale_request.candidate_id, 1)), + ) + self.assertEqual(self.current(), winner.state) + self.assertEqual(self.count("trusted_states"), 2) + self.assertIs(result.candidate.status, CandidateStatus.REJECTED) + with self.uow() as u: + chain = describe_chain(u, stale_request.candidate_id) + self.assertEqual(chain.problems(), []) + self.assertEqual(chain.evidence_kinds()[:4], ["STATE", "IDENTITY", "DEPENDENCY", "IMPACT"]) + + +class TestTheParentMovesWhileDeciding(OrchestratorCase): + """The window between ``decide`` and ``record_decision``: another promotion lands there, and + ``record_decision`` refuses a PROMOTE for a candidate whose parent is no longer current + (SM-010). + The orchestrator turns that refusal into what Doc 09 §25 and Doc 06 §27 require: a stale + rejection, with the state unchanged; it does not let the refusal escape.""" + + def test_a_promotion_that_lands_between_decide_and_record_decision(self) -> None: + import core.application.orchestrator as module + + v0 = self.baseline().state + late = self.request(v0, PATCH_SAFE_DIR, n=2) + real = module.decide + landed: list[Any] = [] + + def decide_then_let_another_promotion_land(*args: Any, **kwargs: Any) -> Any: + outcome = real(*args, **kwargs) + if not landed: + landed.append(None) # first: the nested evaluation passes through here as well + landed[0] = self.orchestrator().evaluate(self.request(v0, PATCH_SAFE_DIR, n=1)) + return outcome + + with mock.patch.object(module, "decide", decide_then_let_another_promotion_land): + result = self.evaluate(late) + self.assertEqual(landed[0].kind, ResultKind.PROMOTED) + self.assertIs(result.kind, ResultKind.STALE) + assert result.decision is not None + self.assertIs(result.decision.reason, DecisionReason.STALE_PARENT) + self.assertEqual(self.current(), landed[0].state) + self.assertEqual(self.count("trusted_states"), 2) + self.assertIs(result.candidate.status, CandidateStatus.REJECTED) + with self.uow() as u: + self.assertEqual(describe_chain(u, late.candidate_id).problems(), []) + + +class TestIdempotency(OrchestratorCase): + def test_the_same_request_again_writes_nothing(self) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + first = self.evaluate(request) + before = dump(self.path) + second = self.evaluate(request) + self.assertEqual(dump(self.path), before) + self.assertIs(second.kind, ResultKind.DUPLICATE) + self.assertTrue(second.duplicate) + self.assertEqual(second.state, first.state) + self.assertEqual(second.decision, first.decision) + + def test_a_rejection_again_is_a_duplicate_with_the_same_kind(self) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_REGRESSION_DIR) + self.evaluate(request) + before = dump(self.path) + again = self.evaluate(request) + self.assertIs(again.kind, ResultKind.REJECTED) + self.assertTrue(again.duplicate) + self.assertEqual(dump(self.path), before) + + def test_another_directory_under_the_same_ids_conflicts_and_changes_nothing(self) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + self.evaluate(request) + before = dump(self.path) + import dataclasses + + with self.assertRaises(ConflictingDuplicateError): + self.evaluate(dataclasses.replace(request, candidate_dir=PATCH_REGRESSION_DIR)) + self.assertEqual(dump(self.path), before) + + def test_another_patch_under_the_same_candidate_conflicts(self) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + self.evaluate(request) + import dataclasses + + other = self.request(v0, PATCH_SAFE_DIR, n=1) + from core.domain.enums import CandidateSource + from core.domain.patch import Patch + + patch = Patch.create( + source=CandidateSource.FIXED_PATCH, + content="a different patch\n", + parent_state_id=v0.state_id, + now=v0.created_at, + patch_id=uid(3999), + ) + with self.assertRaises(ConflictingDuplicateError): + self.evaluate(dataclasses.replace(other, patch=patch)) + + +class TestFailureOutsideDecisions(OrchestratorCase): + def test_an_exception_in_a_stage_propagates_and_leaves_the_candidate_analyzing(self) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + + def boom(name: str) -> None: + if name == "after_impact": + raise InjectedFault(name) + + with self.assertRaises(InjectedFault): + self.evaluate(request, checkpoint=boom) + self.assertEqual(self.current(), v0) + with self.uow() as u: + stored = u.candidates.get(request.candidate_id) + chain = describe_chain(u, request.candidate_id) + assert stored is not None + self.assertIs(stored.status, CandidateStatus.ANALYZING) + self.assertEqual(chain.event_types()[-1], "ERROR_OCCURRED") + self.assertEqual(chain.problems(finished=False), []) + + def test_a_rerun_completes_with_exactly_one_new_state(self) -> None: + v0 = self.baseline().state + request = self.request(v0, PATCH_SAFE_DIR) + + def boom(name: str) -> None: + if name == "after_verification": + raise InjectedFault(name) + + with self.assertRaises(InjectedFault): + self.evaluate(request, checkpoint=boom) + result = self.evaluate(request) + self.assertIs(result.kind, ResultKind.PROMOTED) + self.assertEqual(self.count("trusted_states"), 2) + with self.uow() as u: + self.assertEqual(describe_chain(u, request.candidate_id).problems(), []) + + def test_the_clock_is_the_injected_one(self) -> None: + v0 = self.baseline().state + ticks = {"n": 0} + + def clock() -> Any: + ticks["n"] += 1 + return fixed_clock(60)() + + self.evaluate(self.request(v0, PATCH_SAFE_DIR), clock=clock) + self.assertGreater(ticks["n"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_recorder.py b/tests/unit/test_recorder.py new file mode 100644 index 0000000..29871db --- /dev/null +++ b/tests/unit/test_recorder.py @@ -0,0 +1,564 @@ +"""The orchestrator's derived ids and the stage recorder (P6c step 2; Doc 11 §5, §8-§12, §25, §28, +§38; Doc 06 §28; C-54, C-55, C-170 (4), C-172). + +HAND EXPECTATIONS (written from the P6c prompt and the specs, not from the code): + +* every id is ``uuid5(ORCHESTRATOR_NAMESPACE, "::[:...]")``; + the namespace is the constant below, so a change of it (or of a key) is a visible test failure; +* a stage record is bound to the candidate: ``candidate_id``, ``parent_state_id``, the candidate's + ``state_hash`` and the attempt, run and correlation ids of the context (C-55); its kind is STATE, + IDENTITY, DEPENDENCY, IMPACT or VERIFICATION (one per result); IMPACT is candidate-bound (C-54); +* the audit events are IDENTITY_COMPLETED, DEPENDENCY_COMPLETED, IMPACT_COMPLETED and one + VERIFICATION_COMPLETED per result, each naming the evidence record it belongs to; +* recording the same thing again changes nothing (a duplicate is accepted); another content under + the same id raises ``ConflictingDuplicateError``; a payload with a secret is refused and nothing + is stored. +""" + +from __future__ import annotations + +import unittest +import uuid +from functools import cache +from typing import Any + +from core.application import ids as oids +from core.application.recorder import StageRecorder +from core.domain.audit import AuditEventType +from core.domain.errors import DomainValidationError +from core.domain.evidence import ArtifactRef, EvidenceKind +from core.domain.promotion import ConflictingDuplicateError +from tests.domain_builders import patch_for, uid +from tests.identity_builders import NOW as PIPELINE_NOW +from tests.promotion_commit_builders import CONTEXT, NOW, dump +from tests.promotion_pipeline import PipelineRun, fixtures_with_a_parent, pipeline +from tests.promotion_pipeline_commit import PipelineCommitCase + +SAFE = "canonical_safe_repair_promote_v1" +REGRESSION = "canonical_security_repair_breaks_function_reject_v1" +NAMESPACE = uuid.UUID("5e1d7a43-92c0-4b6f-8d3a-17f4c9e02b68") +CID = uid(3001) +AT = PIPELINE_NOW + + +def without(data: dict[str, Any], key: str) -> dict[str, Any]: + return {k: v for k, v in data.items() if k != key} + + +def expected(key: str) -> str: + return str(uuid.uuid5(NAMESPACE, key)) + + +@cache +def run_of(fixture_id: str) -> PipelineRun: + return pipeline(fixtures_with_a_parent()[fixture_id]) + + +class TestDerivedIds(unittest.TestCase): + def test_the_namespace_is_a_pinned_constant(self) -> None: + self.assertEqual(oids.ORCHESTRATOR_NAMESPACE, NAMESPACE) + + def test_it_is_not_the_namespace_of_the_promotion_package(self) -> None: + from core.promotion.ids import PROMOTION_NAMESPACE + + self.assertNotEqual(oids.ORCHESTRATOR_NAMESPACE, PROMOTION_NAMESPACE) + + def test_the_decision_id(self) -> None: + self.assertEqual(oids.decision_id_for(CID, 1), expected(f"decision:{CID}:1")) + self.assertNotEqual(oids.decision_id_for(CID, 1), oids.decision_id_for(CID, 2)) + self.assertNotEqual(oids.decision_id_for(CID, 1), oids.decision_id_for(uid(3002), 1)) + + def test_the_evidence_id(self) -> None: + self.assertEqual(oids.evidence_id_for(CID, 1, "state"), expected(f"evidence:{CID}:1:state")) + self.assertEqual( + oids.evidence_id_for(CID, 2, "verification", "STRUCTURAL"), + expected(f"evidence:{CID}:2:verification:STRUCTURAL"), + ) + ids = { + oids.evidence_id_for(CID, 1, stage) + for stage in ("state", "identity", "dependency", "impact", "error") + } + self.assertEqual(len(ids), 5) + + def test_the_event_id(self) -> None: + self.assertEqual( + oids.event_id_for(CID, 1, AuditEventType.IMPACT_COMPLETED), + expected(f"event:{CID}:1:IMPACT_COMPLETED"), + ) + self.assertEqual( + oids.event_id_for(CID, 1, AuditEventType.VERIFICATION_COMPLETED, "RESOURCE:a.b"), + expected(f"event:{CID}:1:VERIFICATION_COMPLETED:RESOURCE:a.b"), + ) + + def test_the_verification_id_source_counts_from_zero_and_repeats(self) -> None: + first = oids.verification_ids(CID, 1) + second = oids.verification_ids(CID, 1) + got = [first() for _ in range(3)] + self.assertEqual(got, [expected(f"verification:{CID}:1:{n}") for n in range(3)]) + self.assertEqual([second() for _ in range(3)], got) + other = oids.verification_ids(CID, 2) + self.assertNotEqual(other(), got[0]) + + def test_the_baseline_ids(self) -> None: + base = uid(3100) + self.assertEqual(oids.baseline_state_id_for(base), expected(f"baseline-state:{base}")) + self.assertEqual( + oids.baseline_evidence_id_for(base, "verification", "INVARIANT:INV-SEC-001:1"), + expected(f"baseline-evidence:{base}:verification:INVARIANT:INV-SEC-001:1"), + ) + self.assertEqual( + oids.baseline_event_id_for(base, AuditEventType.STATE_CREATED), + expected(f"baseline-event:{base}:STATE_CREATED"), + ) + source = oids.baseline_verification_ids(base) + self.assertEqual(source(), expected(f"baseline-verification:{base}:0")) + self.assertEqual(source(), expected(f"baseline-verification:{base}:1")) + + def test_bad_arguments_are_refused(self) -> None: + for call in ( + lambda: oids.decision_id_for("not-a-uuid", 1), + lambda: oids.decision_id_for(CID, 0), + lambda: oids.decision_id_for(CID, True), + lambda: oids.decision_id_for(CID, "1"), + lambda: oids.evidence_id_for(CID, 1, "nonsense"), + lambda: oids.evidence_id_for(CID, 1, "state", ""), + lambda: oids.event_id_for(CID, 1, "IMPACT_COMPLETED"), + lambda: oids.verification_ids("x", 1), + lambda: oids.baseline_state_id_for("x"), + ): + with self.assertRaises(DomainValidationError): + call() + + +class RecorderCase(PipelineCommitCase): + def setUp(self) -> None: + super().setUp() + self.run = run_of(SAFE) + self.install(self.run) + self.candidate = self.store(self.run) + self.recorder = StageRecorder(CONTEXT) + + def evidence_rows(self) -> list[Any]: + with self.uow() as u: + return list(u.evidence.list_for_candidate(self.candidate.candidate_id)) + + def events(self) -> list[Any]: + with self.uow() as u: + return list(u.audit.list_for_candidate(self.candidate.candidate_id)) + + def payload(self, evidence_id: str) -> Any: + with self.uow() as u: + event = u.evidence.get(evidence_id) + assert event is not None + return u.evidence.resolve(ArtifactRef.parse(event.payload_ref)) + + def event_payload(self, event: Any) -> Any: + with self.uow() as u: + return u.evidence.resolve(ArtifactRef.parse(event.payload_ref)) + + def record_all(self) -> dict[str, str]: + patch = patch_for(self.run.trusted, n=7) + cid = self.candidate.candidate_id + with self.uow() as u: + self.recorder.record_candidate(u, self.candidate, patch, attempt_number=1, at=AT) + with self.uow() as u: + self.recorder.record_identity( + u, self.candidate, self.run.inputs.identity, attempt_number=1, at=AT, when=NOW + ) + with self.uow() as u: + self.recorder.record_dependency( + u, + self.candidate, + self.run.inputs.candidate_graph, + attempt_number=1, + at=AT, + when=NOW, + ) + with self.uow() as u: + self.recorder.record_impact( + u, self.candidate, self.run.impact, attempt_number=1, at=AT, when=NOW + ) + with self.uow() as u: + found = self.recorder.record_verification( + u, self.candidate, self.run.verification, attempt_number=1, when=NOW + ) + assert cid == self.candidate.candidate_id + return found + + +class TestStageRecords(RecorderCase): + def test_the_state_record_is_bound_to_the_candidate(self) -> None: + patch = patch_for(self.run.trusted, n=7) + with self.uow() as u: + evidence_id = self.recorder.record_candidate( + u, self.candidate, patch, attempt_number=1, at=AT + ) + self.assertEqual(evidence_id, oids.evidence_id_for(self.candidate.candidate_id, 1, "state")) + [row] = self.evidence_rows() + self.assertEqual(row.evidence_id, evidence_id) + self.assertIs(row.kind, EvidenceKind.STATE) + self.assertEqual(row.candidate_id, self.candidate.candidate_id) + self.assertEqual(row.parent_state_id, self.run.trusted.state_id) + self.assertEqual(row.state_hash, self.candidate.state_hash) + self.assertIsNone(row.state_id) + self.assertEqual( + (row.run_id, row.attempt_id, row.correlation_id, row.operation_id), + (CONTEXT.run_id, CONTEXT.attempt_id, CONTEXT.correlation_id, CONTEXT.operation_id), + ) + body = self.payload(evidence_id) + self.assertEqual(body["patch_id"], patch.patch_id) + self.assertEqual(body["patch_hash"], patch.content_hash) + self.assertEqual(body["state_hash"], self.candidate.state_hash) + self.assertEqual(body["parent_state_id"], self.run.trusted.state_id) + + def test_identity_dependency_and_impact_records_and_events(self) -> None: + self.record_all() + kinds = {row.kind: row for row in self.evidence_rows()} + cid = self.candidate.candidate_id + for kind, stage in ( + (EvidenceKind.IDENTITY, "identity"), + (EvidenceKind.DEPENDENCY, "dependency"), + (EvidenceKind.IMPACT, "impact"), + ): + with self.subTest(kind=kind): + row = kinds[kind] + self.assertEqual(row.evidence_id, oids.evidence_id_for(cid, 1, stage)) + self.assertEqual(row.candidate_id, cid) + self.assertEqual(row.parent_state_id, self.run.trusted.state_id) + self.assertEqual(row.state_hash, self.candidate.state_hash) + self.assertEqual(row.attempt_id, CONTEXT.attempt_id) + # the report as the domain writes it, without the random id the engine drew for it + self.assertEqual( + self.payload(kinds[EvidenceKind.IMPACT].evidence_id), + without(self.run.impact.to_dict(), "report_id"), + ) + self.assertEqual( + self.payload(kinds[EvidenceKind.IDENTITY].evidence_id), + without(self.run.inputs.identity.to_dict(), "report_id"), + ) + self.assertEqual( + self.payload(kinds[EvidenceKind.DEPENDENCY].evidence_id), + without(self.run.inputs.candidate_graph.to_dict(), "graph_id"), + ) + wanted = { + AuditEventType.IDENTITY_COMPLETED: EvidenceKind.IDENTITY, + AuditEventType.DEPENDENCY_COMPLETED: EvidenceKind.DEPENDENCY, + AuditEventType.IMPACT_COMPLETED: EvidenceKind.IMPACT, + } + got = {e.event_type: e for e in self.events() if e.event_type in wanted} + self.assertEqual(set(got), set(wanted)) + for event_type, kind in wanted.items(): + self.assertEqual( + self.event_payload(got[event_type])["evidence_id"], kinds[kind].evidence_id + ) + self.assertEqual(got[event_type].candidate_id, cid) + + def test_one_verification_record_and_event_per_result(self) -> None: + found = self.record_all() + results = self.run.verification.results + cid = self.candidate.candidate_id + self.assertEqual(set(found), {r.verification_id for r in results}) + rows = { + r.evidence_id: r for r in self.evidence_rows() if r.kind is EvidenceKind.VERIFICATION + } + self.assertEqual(len(rows), len(results)) + for result in results: + with self.subTest(target=result.target_id): + evidence_id = found[result.verification_id] + self.assertEqual( + evidence_id, oids.evidence_id_for(cid, 1, "verification", result.target_id) + ) + row = rows[evidence_id] + self.assertEqual(row.candidate_id, cid) + self.assertEqual(row.parent_state_id, self.run.trusted.state_id) + self.assertEqual(row.state_hash, self.candidate.state_hash) + self.assertEqual(row.attempt_id, CONTEXT.attempt_id) + self.assertEqual( + self.payload(evidence_id)["verification_id"], result.verification_id + ) + done = [ + self.event_payload(e) + for e in self.events() + if e.event_type is AuditEventType.VERIFICATION_COMPLETED + ] + self.assertEqual( + sorted((p["verification_id"], p["target"], p["result"]) for p in done), + sorted((r.verification_id, r.target_id, r.result.value) for r in results), + ) + for body in done: + self.assertIn(body["evidence_id"], found.values()) + + def test_events_follow_the_stages_in_sequence_order(self) -> None: + self.record_all() + types = [e.event_type for e in self.events()] + n = len(self.run.verification.results) + self.assertEqual( + types, + [ + AuditEventType.IDENTITY_COMPLETED, + AuditEventType.DEPENDENCY_COMPLETED, + AuditEventType.IMPACT_COMPLETED, + *[AuditEventType.VERIFICATION_COMPLETED] * n, + ], + ) + + +class TestIdempotency(RecorderCase): + def test_recording_everything_again_writes_nothing(self) -> None: + first = self.record_all() + before = dump(self.path) + second = self.record_all() + self.assertEqual(first, second) + self.assertEqual(dump(self.path), before) + + def test_a_rerun_of_the_pipeline_records_the_same_payloads(self) -> None: + """R1 of the P6c prompt: the identity, dependency and impact engines draw a new random + ``report_id`` or ``graph_id`` on every run (there is no injection point). Those two ids are + outside the content hash; the recorder leaves them out of the payload, so a rerun that + computes the reports again records duplicates, not conflicts.""" + self.record_all() + before = dump(self.path) + again = pipeline(fixtures_with_a_parent()[SAFE]) + self.assertNotEqual(again.impact.report_id, self.run.impact.report_id) + self.assertNotEqual(again.inputs.identity.report_id, self.run.inputs.identity.report_id) + self.assertNotEqual( + again.inputs.candidate_graph.graph_id, self.run.inputs.candidate_graph.graph_id + ) + self.run = again + self.record_all() + self.assertEqual(dump(self.path), before) + + def test_another_report_under_the_same_ids_raises_and_stores_nothing(self) -> None: + self.record_all() + before = dump(self.path) + other = run_of(REGRESSION) + for call in ( + lambda u: self.recorder.record_identity( + u, self.candidate, other.inputs.identity, attempt_number=1, at=AT, when=NOW + ), + lambda u: self.recorder.record_dependency( + u, self.candidate, other.inputs.candidate_graph, attempt_number=1, at=AT, when=NOW + ), + lambda u: self.recorder.record_impact( + u, self.candidate, other.impact, attempt_number=1, at=AT, when=NOW + ), + ): + with self.assertRaises(ConflictingDuplicateError), self.uow() as u: + call(u) + self.assertEqual(dump(self.path), before) + + def test_another_candidate_under_the_same_state_record_raises(self) -> None: + """The same candidate id and attempt, built from another configuration (another state + hash): the STATE record of the first is not overwritten.""" + self.record_all() + other = run_of(REGRESSION) + self.assertEqual(other.candidate.candidate_id, self.candidate.candidate_id) + self.assertNotEqual(other.candidate.state_hash, self.candidate.state_hash) + patch = patch_for(other.trusted, n=7) + before = dump(self.path) + with self.assertRaises(ConflictingDuplicateError), self.uow() as u: + self.recorder.record_candidate(u, other.candidate, patch, attempt_number=1, at=AT) + self.assertEqual(dump(self.path), before) + + def test_the_same_candidate_in_another_attempt_is_a_different_record(self) -> None: + self.record_all() + with self.uow() as u: + self.recorder.record_impact( + u, self.candidate, self.run.impact, attempt_number=2, at=AT, when=NOW + ) + impact = [r for r in self.evidence_rows() if r.kind is EvidenceKind.IMPACT] + self.assertEqual(len(impact), 2) + + +class TestEmit(RecorderCase): + def event_args(self) -> dict[str, Any]: + return { + "event_id": oids.event_id_for( + self.candidate.candidate_id, 1, AuditEventType.CANDIDATE_CREATED + ), + "payload": { + "parent_state_id": self.run.trusted.state_id, + "patch_id": uid(77), + "source": "FIXED_PATCH", + }, + "when": NOW, + "candidate_id": self.candidate.candidate_id, + } + + def test_emit_appends_once(self) -> None: + with self.uow() as u: + self.recorder.emit(u, AuditEventType.CANDIDATE_CREATED, **self.event_args()) + before = dump(self.path) + with self.uow() as u: + self.recorder.emit(u, AuditEventType.CANDIDATE_CREATED, **self.event_args()) + self.assertEqual(dump(self.path), before) + [event] = self.events() + self.assertIs(event.event_type, AuditEventType.CANDIDATE_CREATED) + self.assertEqual(event.correlation_id, CONTEXT.correlation_id) + self.assertEqual(event.operation_id, CONTEXT.operation_id) + + def test_another_payload_under_the_same_event_id_raises(self) -> None: + with self.uow() as u: + self.recorder.emit(u, AuditEventType.CANDIDATE_CREATED, **self.event_args()) + changed = self.event_args() + changed["payload"] = {**changed["payload"], "patch_id": uid(78)} + before = dump(self.path) + with self.assertRaises(ConflictingDuplicateError), self.uow() as u: + self.recorder.emit(u, AuditEventType.CANDIDATE_CREATED, **changed) + self.assertEqual(dump(self.path), before) + + def test_a_missing_required_payload_key_is_refused(self) -> None: + args = self.event_args() + args["payload"] = {"parent_state_id": self.run.trusted.state_id} + with self.assertRaises(DomainValidationError), self.uow() as u: + self.recorder.emit(u, AuditEventType.CANDIDATE_CREATED, **args) + self.assertEqual(self.events(), []) + + def test_a_payload_with_a_secret_is_refused_not_stored(self) -> None: + args = self.event_args() + args["payload"] = {**args["payload"], "note": "AKIA" + "IOSFODNN7" + "EXAMPLE"} + before = dump(self.path) + with self.assertRaises(DomainValidationError), self.uow() as u: + self.recorder.emit(u, AuditEventType.CANDIDATE_CREATED, **args) + self.assertEqual(dump(self.path), before) + + +class TestErrorRecords(RecorderCase): + def test_a_candidate_bound_error_record_and_its_event(self) -> None: + with self.uow() as u: + evidence_id = self.recorder.record_error( + u, + stage="normalize", + error_type="ConstructionFailed", + detail={"status": "INVALID", "diagnostics": ["x.tf: bad block"]}, + attempt_number=1, + at=AT, + when=NOW, + candidate=self.candidate, + ) + self.assertEqual(evidence_id, oids.evidence_id_for(self.candidate.candidate_id, 1, "error")) + [row] = self.evidence_rows() + self.assertIs(row.kind, EvidenceKind.ERROR) + self.assertEqual(row.candidate_id, self.candidate.candidate_id) + self.assertEqual(row.parent_state_id, self.run.trusted.state_id) + self.assertEqual(row.attempt_id, CONTEXT.attempt_id) + [event] = self.events() + self.assertIs(event.event_type, AuditEventType.ERROR_OCCURRED) + body = self.event_payload(event) + self.assertEqual(body["evidence_id"], evidence_id) + self.assertEqual(body["stage"], "normalize") + self.assertEqual(body["error_type"], "ConstructionFailed") + + def test_the_error_record_of_a_failed_candidate_is_unbound_and_names_the_candidate( + self, + ) -> None: + """A candidate that never got a state hash cannot bind evidence (Doc 11 §44): the ERROR + record is unbound, its payload and its audit event name the candidate.""" + from core.domain.state import CandidateState, fail_building, start_building + from tests.domain_builders import patch_for as make_patch + + trusted = self.run.trusted + created = CandidateState.create( + parent=trusted, + patch=make_patch(trusted, n=21), + candidate_sequence=5, + now=AT, + candidate_id=uid(3500), + ) + failed = fail_building(start_building(created), "the configuration does not parse") + self.assertIsNone(failed.state_hash) + with self.uow() as u: + u.patches.save(make_patch(trusted, n=21)) + u.candidates.create(created) + u.candidates.save_transition(start_building(created)) + u.candidates.save_transition(failed) + evidence_id = self.recorder.record_error( + u, + stage="normalize", + error_type="ConstructionFailed", + detail={"status": "INVALID"}, + attempt_number=1, + at=AT, + when=NOW, + candidate=failed, + ) + row = u.evidence.get(evidence_id) + events = u.audit.list_for_candidate(failed.candidate_id) + body = u.evidence.resolve(ArtifactRef.parse(row.payload_ref)) if row else None + assert row is not None + self.assertIs(row.kind, EvidenceKind.ERROR) + self.assertEqual( + (row.candidate_id, row.state_id, row.parent_state_id, row.state_hash), + (None, None, None, None), + ) + self.assertEqual(body["candidate_id"], failed.candidate_id) + self.assertEqual([e.event_type for e in events], [AuditEventType.ERROR_OCCURRED]) + + def test_an_unbound_error_record_has_no_state_and_no_candidate(self) -> None: + base = uid(3100) + with self.uow() as u: + evidence_id = self.recorder.record_error( + u, + stage="baseline", + error_type="BaselineRefused", + detail={"reasons": ["not eligible"]}, + attempt_number=1, + at=AT, + when=NOW, + evidence_id=oids.baseline_evidence_id_for(base, "error"), + event_id=oids.baseline_event_id_for(base, AuditEventType.ERROR_OCCURRED), + ) + row = u.evidence.get(evidence_id) + assert row is not None + self.assertEqual( + (row.candidate_id, row.state_id, row.parent_state_id, row.state_hash), + (None, None, None, None), + ) + + def test_an_error_detail_with_a_secret_is_refused(self) -> None: + before = dump(self.path) + with self.assertRaises(DomainValidationError), self.uow() as u: + self.recorder.record_error( + u, + stage="normalize", + error_type="X", + detail={"message": "key " + "AKIA" + "IOSFODNN7" + "EXAMPLE"}, + attempt_number=1, + at=AT, + when=NOW, + candidate=self.candidate, + ) + self.assertEqual(dump(self.path), before) + + +class TestArguments(RecorderCase): + def test_wrong_types_are_refused(self) -> None: + with self.assertRaises(DomainValidationError): + StageRecorder("context") # type: ignore[arg-type] + with self.assertRaises(DomainValidationError), self.uow() as u: + self.recorder.record_impact( + u, self.candidate, {"x": 1}, attempt_number=1, at=AT, when=NOW # type: ignore[arg-type] + ) + with self.assertRaises(DomainValidationError), self.uow() as u: + self.recorder.record_impact( + u, "candidate", self.run.impact, attempt_number=1, at=AT, when=NOW # type: ignore[arg-type] + ) + + def test_a_report_for_another_candidate_is_refused(self) -> None: + other = run_of(REGRESSION) + self.assertNotEqual(other.impact.candidate_id, "") + different = pipeline( + fixtures_with_a_parent()[SAFE], candidate_id=uid(3999), sequence=2, patch_n=9 + ) + with self.assertRaises(DomainValidationError), self.uow() as u: + self.recorder.record_impact( + u, self.candidate, different.impact, attempt_number=1, at=AT, when=NOW + ) + with self.assertRaises(DomainValidationError), self.uow() as u: + self.recorder.record_verification( + u, self.candidate, different.verification, attempt_number=1, when=NOW + ) + + +if __name__ == "__main__": + unittest.main()