diff --git a/core/application/baseline.py b/core/application/baseline.py index 9a2ee5f..410952a 100644 --- a/core/application/baseline.py +++ b/core/application/baseline.py @@ -95,7 +95,9 @@ def __init__(self, reasons: Sequence[str]) -> None: @dataclass(frozen=True) class BaselineRequest: """What to establish. ``eligible`` is the caller's declaration that the fixture may serve as a - baseline (Doc 13 §13); ``now`` is the one instant used for everything that is hashed.""" + baseline (Doc 13 §13); ``now`` is the one instant used for everything that is hashed. + ``allow_no_invariants`` (default off) accepts an empty ``definitions``: only the experiment + harness sets it, for condition A on a change request (C-206); the gates still run.""" baseline_id: str lineage_id: str @@ -108,6 +110,7 @@ class BaselineRequest: eligible: bool verifiers: VerifierTable = DEFAULT_VERIFIERS registry_version: int = 1 + allow_no_invariants: bool = False def __post_init__(self) -> None: require_uuid(self.baseline_id, "BaselineRequest.baseline_id") @@ -118,8 +121,12 @@ def __post_init__(self) -> None: raise DomainValidationError("BaselineRequest.eligible must be a bool") if not isinstance(self.context, PromotionContext): raise DomainValidationError("BaselineRequest.context must be a PromotionContext") + if not isinstance(self.allow_no_invariants, bool): + raise DomainValidationError("BaselineRequest.allow_no_invariants must be a bool") definitions = tuple(self.definitions) - if not definitions or not all(isinstance(d, Invariant) for d in definitions): + if (not definitions and not self.allow_no_invariants) or not all( + isinstance(d, Invariant) for d in definitions + ): raise DomainValidationError("BaselineRequest.definitions must hold Invariant objects") bindings = tuple(self.bindings) if not all(isinstance(b, SemanticBinding) for b in bindings): @@ -353,6 +360,7 @@ def _write( invariant_registry_version=request.registry_version, now=request.now, state_id=prepared.state_id, + allow_no_invariants=request.allow_no_invariants, ) if existing is not None and ( existing.state_hash != state.state_hash or existing.lineage_id != state.lineage_id diff --git a/core/application/orchestrator.py b/core/application/orchestrator.py index 6b660f9..b46e291 100644 --- a/core/application/orchestrator.py +++ b/core/application/orchestrator.py @@ -31,6 +31,13 @@ back with ``duplicate=True``. ``checkpoint`` is a test-only hook, called with the name of a stage after its records committed. + +**Experiment seams (P8a; C-196, C-198).** ``identity_strategy`` is the identity strategy given to +``compare``; its default is ``DEFAULT_STRATEGY`` (``EvidenceIdentity``), the only one production +code uses, and only the experiment harness passes the address-only ablation (C-127, condition C'). +``EvaluationRequest.open_targets`` (default none) is handed to ``decide``; ``check_open_targets`` +refuses a bad one against the candidate's parent before anything is stored. The result carries the +identity report and the work the scope policy selected, for the experiment records. """ from __future__ import annotations @@ -52,8 +59,9 @@ from core.domain.enums import CandidateSource, CandidateStatus, DecisionType from core.domain.errors import DomainValidationError, StaleParentError from core.domain.hashing import content_hash +from core.domain.identity import IdentityReport from core.domain.ids import require_uuid -from core.domain.impact import ImpactReport +from core.domain.impact import ImpactReport, VerificationWorkItem from core.domain.invariant import Invariant from core.domain.patch import Patch from core.domain.promotion import ( @@ -75,6 +83,7 @@ ) from core.domain.verification import ConstructionStatus, VerificationReport from core.identity.engine import compare +from core.identity.strategy import DEFAULT_STRATEGY, IdentityStrategy from core.impact.engine import compute_impact from core.impact.inputs import PriorProof from core.promotion import ids as pids @@ -85,7 +94,7 @@ commit_promotion, record_decision, ) -from core.promotion.decide import decide +from core.promotion.decide import check_open_targets, decide from core.promotion.policy import dail_policy from core.terraform_model.normalization import normalize_directory from core.verification.dispatch import DEFAULT_VERIFIERS, VerifierTable @@ -141,6 +150,7 @@ class EvaluationRequest: bindings: tuple[SemanticBinding, ...] context: PromotionContext oracle_verdicts: tuple[OracleVerdict, ...] = () + open_targets: tuple[str, ...] = () _PATCH_SOURCE: ClassVar[CandidateSource] = CandidateSource.FIXED_PATCH def __post_init__(self) -> None: @@ -167,6 +177,12 @@ def __post_init__(self) -> None: verdicts = tuple(self.oracle_verdicts) if not all(isinstance(v, OracleVerdict) for v in verdicts): raise EvaluationInputError("EvaluationRequest.oracle_verdicts must hold OracleVerdict") + if isinstance(self.open_targets, str): + raise EvaluationInputError("EvaluationRequest.open_targets must be a list of ids") + opened = tuple(self.open_targets) + if not all(isinstance(t, str) and t for t in opened) or len(set(opened)) != len(opened): + raise EvaluationInputError("EvaluationRequest.open_targets: invariant ids, each once") + object.__setattr__(self, "open_targets", opened) object.__setattr__(self, "definitions", definitions) object.__setattr__(self, "bindings", bindings) object.__setattr__(self, "oracle_verdicts", verdicts) @@ -222,6 +238,8 @@ class EvaluationResult: verification: VerificationReport | None = None evidence_ids: Mapping[str, str] | None = None duplicate: bool = False + identity: IdentityReport | None = None + work: tuple[VerificationWorkItem, ...] | None = None class EvaluationOrchestrator: @@ -238,6 +256,7 @@ def __init__( verifiers: VerifierTable = DEFAULT_VERIFIERS, policy_factory: Callable[..., PromotionPolicy] = dail_policy, checkpoint: Callable[[str], None] | None = None, + identity_strategy: IdentityStrategy = DEFAULT_STRATEGY, ) -> None: if not callable(factory): raise EvaluationInputError("EvaluationOrchestrator: factory must open a unit of work") @@ -249,6 +268,12 @@ def __init__( raise EvaluationInputError("EvaluationOrchestrator: policy_version must be text") if checkpoint is not None and not callable(checkpoint): raise EvaluationInputError("EvaluationOrchestrator: checkpoint must be callable") + if not callable(getattr(identity_strategy, "decide", None)) or not isinstance( + getattr(identity_strategy, "algorithm_version", None), str + ): + raise EvaluationInputError( + "EvaluationOrchestrator: identity_strategy must be an IdentityStrategy" + ) try: flag = config.get(_FLAG) except (KeyError, AttributeError) as missing: @@ -263,6 +288,14 @@ def __init__( self._policy_factory = policy_factory self._checkpoint = checkpoint self._automatic = flag + self._identity = identity_strategy + + def _policy(self, request: EvaluationRequest) -> PromotionPolicy: + return self._policy_factory( + policy_version=self._policy_version, + invariants=request.definitions, + automatic_enabled=self._automatic, + ) def evaluate(self, request: EvaluationRequest) -> EvaluationResult: """Evaluate ``request`` (see the module docstring). Raises ``EvaluationInputError`` for a @@ -296,6 +329,8 @@ def __init__(self, orchestrator: EvaluationOrchestrator, request: EvaluationRequ self.stage = "load" self.candidate: CandidateState | None = None self.failed_now = False # this call is the one that moved the candidate to FAILED + self.identity: IdentityReport | None = None + self.work: tuple[VerificationWorkItem, ...] | None = None # --- small helpers ------------------------------------------------------------------------- @@ -334,6 +369,7 @@ def execute(self) -> EvaluationResult: f"the patch was written against state {r.patch.parent_state_id}, which is not a " f"state of lineage {r.lineage_id}" ) + check_open_targets(parent, o._policy(r), r.open_targets) # C-196, before any write self.stage = "candidate" candidate = self._create(parent) at = candidate.created_at @@ -353,7 +389,9 @@ def execute(self) -> EvaluationResult: parent_state_id=parent.state_id, candidate_id=self._cid, now=at, + strategy=o._identity, ) + self.identity = identity self._record( lambda u: self.recorder.record_identity( u, candidate, identity, attempt_number=r.attempt_number, at=at, when=self._when() @@ -398,7 +436,8 @@ def execute(self) -> EvaluationResult: ) self._passed("after_impact") self.stage = "verification" - work = o._scope_policy.select(impact, r.definitions) + work = o._scope_policy.select(impact, r.definitions, parent_refs=parent.invariant_refs) + self.work = tuple(work) inputs = VerificationInputs( candidate_resources=outcome.resources, parent_resources=parent.resources, @@ -634,6 +673,7 @@ def _decision( now=at, attempt_number=r.attempt_number, oracle_verdicts=r.oracle_verdicts, + open_targets=() if stale else r.open_targets, ) return outcome, stale @@ -648,11 +688,7 @@ def _decide( candidate = self._stored() if candidate.status in _TERMINAL: return self._stored_result(candidate, impact, report, evidence_ids) - policy = o._policy_factory( - policy_version=o._policy_version, - invariants=r.definitions, - automatic_enabled=o._automatic, - ) + policy = o._policy(r) outcome, stale = self._decision(candidate, impact, report, policy, at) try: with o._factory() as uow: @@ -675,6 +711,8 @@ def _decide( impact=impact, verification=report, evidence_ids=evidence_ids, + identity=self.identity, + work=self.work, ) self.stage = "commit" with o._factory() as uow: @@ -735,6 +773,8 @@ def _after_commit( verification=report, evidence_ids=evidence_ids, duplicate=committed.duplicate, + identity=self.identity, + work=self.work, ) def _stored_result( @@ -772,6 +812,8 @@ def _stored_result( verification=report, evidence_ids=evidence_ids, duplicate=True, + identity=self.identity, + work=self.work, ) # --- failures that are not decisions ------------------------------------------------------- diff --git a/core/application/scope_policy.py b/core/application/scope_policy.py index 3e98cb8..787324b 100644 --- a/core/application/scope_policy.py +++ b/core/application/scope_policy.py @@ -1,23 +1,54 @@ -"""Verification scope policies (Doc 03 §24; Doc 08 §26; C-142; P4 step 8). +"""Verification scope policies (Doc 03 §24; Doc 08 §26; C-142; P4 step 8; P8a: C-198; P8a-2: +C-210, C-211). A scope policy decides which verification work a candidate owes, given the impact report. It is an experiment-level choice (the conditions of Doc 13), so it lives in the application layer and never in the domain: ``core.domain`` and ``core.impact`` do not import it (rule R2). -``DailScopePolicy`` is the DAIL condition: the verification work is exactly what impact derived, -the work set of the impacted invariants plus the baseline checks of the created resources, all -BLOCKING (Doc 08 §26). The two comparison policies belong to the experiment harness and are built in -P8, not here: *Stateful Full* (every protected invariant is checked again) and *Stateless* (checked -again with no parent state). +``DailScopePolicy`` is the DAIL condition (C and C'): the impact work set (the impacted invariants +and the baseline checks of the created resources, all BLOCKING, Doc 08 §26), plus, since P8a-2 +(C-210), one invariant item for every invariant whose reference in the parent is not PROTECTED +(VIOLATED or UNCERTAIN) and that impact did not already list: the U_t of the paper's required set +R(C_t) = G_gate ∪ U_t ∪ I_t ∪ B. C-159 already required a fresh result for each of them (a VIOLATED +or UNCERTAIN reference is never carried forward); the work set did not schedule one. With every +parent reference PROTECTED the work set is impact's, unchanged. + +The two comparison policies of the experiment harness (P8a, C-198): + +* ``StatefulFullScopePolicy`` (condition B): every protected invariant is checked again, together + with every RESOURCE baseline item of the impact work set. +* ``StatelessScopePolicy`` (condition A): the repair target's invariant item only, or nothing for a + change request (C-197). No baseline item. + +The two gate items (STRUCTURAL, CONSISTENCY) are not selected by any policy: the verification engine +adds them to every work set (C-147). + +**The parent's statuses.** ``select`` takes them as ``parent_refs`` (keyword, default none): the +parent state's invariant references, which the orchestrator already holds. It is the narrowest input +that carries the statuses (C-210); the impact report does not, and the scope policy must not read +the database. Without it ``DailScopePolicy`` returns impact's work set as before. + +An item a policy adds for an invariant impact did not list has the fields impact gives an invariant +item (target, BLOCKING, L1) and the definition's scope (resources, else resource types; the +verifiers do not read the scope). Its reason (C-211) is ``CONDITION_SCOPE`` when the condition's +policy requires it (B, A) and ``NOT_PROTECTED_IN_PARENT`` for the U_t items. """ from __future__ import annotations -from collections.abc import Sequence -from typing import Protocol, runtime_checkable +from collections.abc import Iterable, Sequence +from typing import ClassVar, Protocol, runtime_checkable -from core.domain.impact import ImpactReport, VerificationWorkItem -from core.domain.invariant import Invariant +from core.domain.enums import InvariantStatus +from core.domain.errors import DomainValidationError +from core.domain.impact import ( + ImpactReport, + ReasonCode, + VerificationWorkItem, + WorkPriority, + WorkTargetType, +) +from core.domain.invariant import Invariant, InvariantRef @runtime_checkable @@ -25,17 +56,117 @@ class VerificationScopePolicy(Protocol): """Which verification work a candidate owes.""" def select( - self, report: ImpactReport, protected: Sequence[Invariant] + self, + report: ImpactReport, + protected: Sequence[Invariant], + *, + parent_refs: Sequence[InvariantRef] = (), ) -> tuple[VerificationWorkItem, ...]: - """The work items to run for ``report``, given the protected invariants.""" + """The work items to run for ``report``, given the protected invariants and the parent + state's invariant references.""" ... +def _target(definition: Invariant) -> str: + return f"INVARIANT:{definition.invariant_id}:{definition.version}" + + +def policy_item(definition: Invariant, reason: ReasonCode) -> VerificationWorkItem: + """A work item for ``definition`` that impact did not list (see the module docstring).""" + scope = definition.scope.resources or definition.scope.resource_types + return VerificationWorkItem( + target_id=_target(definition), + target_type=WorkTargetType.INVARIANT, + reason=reason, + scope=scope, + required_evidence="L1", + priority=WorkPriority.BLOCKING, + ) + + +def _ordered(items: Iterable[VerificationWorkItem]) -> tuple[VerificationWorkItem, ...]: + return tuple(sorted(items, key=VerificationWorkItem.sort_key)) + + class DailScopePolicy: - """The DAIL condition: the impact work set, unchanged.""" + """The DAIL condition: the impact work set and the U_t items (C-210).""" + + version: ClassVar[str] = "dail-scope-2" + + def select( + self, + report: ImpactReport, + protected: Sequence[Invariant], + *, + parent_refs: Sequence[InvariantRef] = (), + ) -> tuple[VerificationWorkItem, ...]: + by_id = {d.invariant_id: d for d in protected} + listed = {item.target_id for item in report.verification_work} + added = {} + for ref in parent_refs: + definition = by_id.get(ref.invariant_id) + if ref.status is InvariantStatus.PROTECTED or definition is None: + continue # an undefined one is the decision's policy condition (C-41) + if _target(definition) not in listed: + added[_target(definition)] = policy_item( + definition, ReasonCode.NOT_PROTECTED_IN_PARENT + ) + if not added: + return report.verification_work + return _ordered([*report.verification_work, *added.values()]) + + +class StatefulFullScopePolicy: + """Condition B, Stateful Full (C-198): every protected invariant and every RESOURCE baseline + item of the impact work set. Impact's own item is kept where it listed one.""" + + version: ClassVar[str] = "stateful-full-scope-2" + + def select( + self, + report: ImpactReport, + protected: Sequence[Invariant], + *, + parent_refs: Sequence[InvariantRef] = (), + ) -> tuple[VerificationWorkItem, ...]: + del parent_refs # every protected invariant is checked anyway + listed = {item.target_id: item for item in report.verification_work} + chosen = [ + item for item in report.verification_work if item.target_type is WorkTargetType.RESOURCE + ] + chosen += [ + listed.get(_target(d)) or policy_item(d, ReasonCode.CONDITION_SCOPE) for d in protected + ] + return _ordered(chosen) + + +class StatelessScopePolicy: + """Condition A, Stateless (C-198): the invariant items of ``targets`` only (the repair target, + or none for a change request). Impact's own item is kept where it listed one.""" + + version: ClassVar[str] = "stateless-scope-2" + + def __init__(self, targets: Iterable[str]) -> None: + if isinstance(targets, str): + raise DomainValidationError("StatelessScopePolicy: targets is a list of invariant ids") + self.targets = tuple(sorted(set(targets))) def select( - self, report: ImpactReport, protected: Sequence[Invariant] + self, + report: ImpactReport, + protected: Sequence[Invariant], + *, + parent_refs: Sequence[InvariantRef] = (), ) -> tuple[VerificationWorkItem, ...]: - del protected # impact already derived its work from the protected invariants - return report.verification_work + del parent_refs # A uses no ledger (C-198): its lineage holds only its targets anyway + by_id = {d.invariant_id: d for d in protected} + missing = [t for t in self.targets if t not in by_id] + if missing: + raise DomainValidationError( + f"StatelessScopePolicy: no definition for {', '.join(missing)}" + ) + listed = {item.target_id: item for item in report.verification_work} + return _ordered( + listed.get(_target(by_id[t])) or policy_item(by_id[t], ReasonCode.CONDITION_SCOPE) + for t in self.targets + ) diff --git a/core/domain/impact.py b/core/domain/impact.py index dc1d2bd..2f961d6 100644 --- a/core/domain/impact.py +++ b/core/domain/impact.py @@ -82,6 +82,9 @@ class ReasonCode(StrEnum): NO_PRIOR_EVIDENCE = "NO_PRIOR_EVIDENCE" INVARIANT_VERSION_CHANGED = "INVARIANT_VERSION_CHANGED" VERIFIER_CHANGED = "VERIFIER_CHANGED" + # C-211 (P8a-2): the reasons of invariant work that a scope policy adds; impact never uses them + CONDITION_SCOPE = "CONDITION_SCOPE" + NOT_PROTECTED_IN_PARENT = "NOT_PROTECTED_IN_PARENT" AFFECTED_CODES: Final = frozenset( @@ -105,6 +108,7 @@ class ReasonCode(StrEnum): UNCERTAINTY_CODES: Final = frozenset( {ReasonCode.UNRESOLVED_CONTENT, ReasonCode.WALK_INCOMPLETE, ReasonCode.ENDPOINT_ABSENT} ) +POLICY_CODES: Final = frozenset({ReasonCode.CONDITION_SCOPE, ReasonCode.NOT_PROTECTED_IN_PARENT}) UNAFFECTED_CODES: Final = frozenset( { ReasonCode.NO_CHANGE_IN_FOOTPRINT, @@ -136,6 +140,12 @@ class ReasonCode(StrEnum): ReasonCode.NO_PRIOR_EVIDENCE: "No prior proof exists to carry forward.", ReasonCode.INVARIANT_VERSION_CHANGED: "Only a proof of another invariant version exists.", ReasonCode.VERIFIER_CHANGED: "The prior proof was made by another verifier or version.", + ReasonCode.CONDITION_SCOPE: ( + "The experiment condition's scope policy requires this check (not impact's work)." + ), + ReasonCode.NOT_PROTECTED_IN_PARENT: ( + "The invariant is not PROTECTED in the parent state, so it is checked again (C-210)." + ), } ) @@ -757,7 +767,7 @@ def __post_init__(self) -> None: raise DomainValidationError( "VerificationWorkItem: CREATED_BASELINE is exactly the reason of resource work" ) - if not is_baseline and self.reason not in AFFECTED_CODES | UNCERTAIN_CODES: + if not is_baseline and self.reason not in AFFECTED_CODES | UNCERTAIN_CODES | POLICY_CODES: raise DomainValidationError(f"VerificationWorkItem: bad reason {self.reason.value}") def sort_key(self) -> tuple[str, str]: @@ -931,6 +941,11 @@ def _check_consistency( seen_invariants: set[tuple[str, int]] = set() seen_resources: set[str] = set() for item in work: + if item.reason in POLICY_CODES: + raise DomainValidationError( + f"ImpactReport.verification_work: {item.target_id} has the scope-policy reason " + f"{item.reason.value}, which impact never gives (C-211)" + ) if item.target_type is WorkTargetType.INVARIANT: _, _, rest = item.target_id.partition(":") invariant_id, _, version = rest.rpartition(":") diff --git a/core/domain/promotion.py b/core/domain/promotion.py index 45dcbfa..59bc187 100644 --- a/core/domain/promotion.py +++ b/core/domain/promotion.py @@ -486,7 +486,11 @@ def from_dict(cls, data: Mapping[str, Any]) -> PromotionDecision: @dataclass(frozen=True) class PromotionEvidence: """What a decision rests on (Doc 11 §26). A PROMOTE references every mandatory verification - result used; ``new_state_id`` stays null until P6b creates the state.""" + result used; ``new_state_id`` stays null until P6b creates the state. + + ``open_targets`` (C-196) are the invariants whose FAIL did not block this decision; they are + written to the record (and so to its hash) only when there is one, so a decision without an + open target has exactly the record it had before P8a.""" decision_id: str candidate_id: str @@ -499,10 +503,12 @@ class PromotionEvidence: reason: DecisionReason new_state_id: str | None content_hash: str + open_targets: tuple[str, ...] = () _token: InitVar[object | None] = None def __post_init__(self, _token: object | None) -> None: name = "PromotionEvidence" + object.__setattr__(self, "open_targets", _texts(self.open_targets, f"{name}.open_targets")) require_uuid(self.decision_id, f"{name}.decision_id") require_uuid(self.candidate_id, f"{name}.candidate_id") require_uuid(self.parent_state_id, f"{name}.parent_state_id") @@ -576,8 +582,10 @@ def create( policy_version: str, decision: DecisionType, reason: DecisionReason, + open_targets: Iterable[str] = (), ) -> PromotionEvidence: - probe = { + opened = _texts(open_targets, "PromotionEvidence.open_targets") + probe: dict[str, Any] = { "candidate_id": candidate_id, "parent_state_id": parent_state_id, "required_checks": [c.to_dict() for c in required_checks], @@ -588,6 +596,8 @@ def create( "reason": getattr(reason, "value", reason), "new_state_id": None, } + if opened: + probe["open_targets"] = list(opened) return cls( decision_id=decision_id, candidate_id=candidate_id, @@ -600,10 +610,11 @@ def create( reason=reason, new_state_id=None, content_hash=content_hash(probe), + open_targets=opened, ) def to_dict(self) -> dict[str, Any]: - return { + data: dict[str, Any] = { "decision_id": self.decision_id, "candidate_id": self.candidate_id, "parent_state_id": self.parent_state_id, @@ -616,10 +627,19 @@ def to_dict(self) -> dict[str, Any]: "new_state_id": self.new_state_id, "content_hash": self.content_hash, } + if self.open_targets: + data["open_targets"] = list(self.open_targets) + return data @classmethod def from_dict(cls, data: Mapping[str, Any]) -> PromotionEvidence: - fields = require_keys(data, _EVIDENCE_KEYS, "PromotionEvidence") + keys = (*_EVIDENCE_KEYS, "open_targets") if "open_targets" in data else _EVIDENCE_KEYS + fields = require_keys(data, keys, "PromotionEvidence") + opened = _list_of(fields["open_targets"], "open_targets") if "open_targets" in data else [] + if "open_targets" in data and not opened: + raise DomainValidationError( + "PromotionEvidence.open_targets: an empty list is written as no field" + ) return cls( decision_id=fields["decision_id"], candidate_id=fields["candidate_id"], @@ -635,6 +655,7 @@ def from_dict(cls, data: Mapping[str, Any]) -> PromotionEvidence: reason=parse_enum(fields["reason"], DecisionReason, "reason"), new_state_id=fields["new_state_id"], content_hash=fields["content_hash"], + open_targets=tuple(opened), _token=_COMMIT_TOKEN, ) @@ -746,6 +767,10 @@ class PromotionPolicy: automatic_enabled: bool accept_oracle_for_unknown: bool gate_verifiers: tuple[tuple[str, str, str], ...] + # C-205 (condition A of the experiments): when True the required set is exactly the gate items + # and ``extra_required``; no RESOURCE baseline item, and an invariant of the current state + # outside the set is a policy condition (ESCALATE), never dropped. Default: the C-159 set. + explicit_required_set: bool = False _by_id: dict[str, Invariant] = field( init=False, repr=False, compare=False, default_factory=dict ) @@ -775,6 +800,7 @@ def __post_init__(self) -> None: ) _bool(self.carry_forward_enabled, f"{name}.carry_forward_enabled") _bool(self.automatic_enabled, f"{name}.automatic_enabled") + _bool(self.explicit_required_set, f"{name}.explicit_required_set") if _bool(self.accept_oracle_for_unknown, f"{name}.accept_oracle_for_unknown"): raise DomainValidationError( f"{name}: oracle evidence is never proof, so accept_oracle_for_unknown must be " diff --git a/core/domain/state.py b/core/domain/state.py index c9a037c..63cb53e 100644 --- a/core/domain/state.py +++ b/core/domain/state.py @@ -663,11 +663,17 @@ def establish_baseline( invariant_registry_version: int, now: datetime, state_id: str | None = None, + allow_no_invariants: bool = False, ) -> TrustedState: """The baseline, version 0: no parent and no promotion decision (C-32). It is created only - when the baseline protocol accepts it, with current invariant evidence (Doc 06 §24).""" + when the baseline protocol accepts it, with current invariant evidence (Doc 06 §24). + + ``allow_no_invariants`` (default off) lets the invariant set be empty: only the experiment + harness sets it, for condition A on a change request (C-206).""" + if not isinstance(allow_no_invariants, bool): + raise DomainValidationError("establish_baseline: allow_no_invariants must be a bool") proofs = list(invariant_proofs) - if not proofs: + if not proofs and not allow_no_invariants: raise DomainValidationError( "establish_baseline: at least one invariant proof is required (Doc 06 §24)" ) diff --git a/core/promotion/binding.py b/core/promotion/binding.py index cf2b426..87cada8 100644 --- a/core/promotion/binding.py +++ b/core/promotion/binding.py @@ -20,7 +20,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from typing import Any, Final from core.domain import hashing @@ -106,9 +106,12 @@ def verified_problems( definition: Invariant | None, result: VerificationResultRecord | None, evidence_id: str | None, + is_open: bool = False, ) -> list[str]: - """Why the evidence of a VERIFIED entry is not proof for this candidate (C-164, C-68).""" + """Why the evidence of a VERIFIED entry is not proof for this candidate (C-164, C-68). For an + open target (C-196) the result it names is a FAIL; for anything else, a PASS.""" label = f"{entry.target_id} (VERIFIED)" + wanted = VerificationResult.FAIL if is_open else VerificationResult.PASS if candidate.state_hash is None: return [f"{label}: the candidate has no state_hash"] if result is None or evidence_id is None: @@ -116,11 +119,8 @@ def verified_problems( if definition is None or definition.version != entry.invariant_version: return [f"{label}: no definition of that version to check the verifier against"] problems: list[str] = [] - if ( - result.result is not VerificationResult.PASS - or result.candidate_hash != candidate.state_hash - ): - problems.append(f"{label}: the result is not a PASS of this candidate's hash") + if result.result is not wanted or result.candidate_hash != candidate.state_hash: + problems.append(f"{label}: the result is not a {wanted.value} of this candidate's hash") event, problems_read = _fetch(evidence, evidence_id, label) problems += problems_read if event is None: @@ -144,7 +144,7 @@ def verified_problems( "verifier_version": definition.verifier_version, "candidate_id": candidate.candidate_id, "candidate_hash": candidate.state_hash, - "result": VerificationResult.PASS.value, + "result": wanted.value, } for key, expected in subject.items(): if payload.get(key) != expected: @@ -224,8 +224,10 @@ def binding_problems( definitions: Mapping[str, Invariant], results: Mapping[str, VerificationResultRecord], evidence_ids: Mapping[str, str], + open_targets: Collection[str] = (), ) -> list[str]: - """Every reason the evidence of ``plan`` does not bind to the new state (empty: it does).""" + """Every reason the evidence of ``plan`` does not bind to the new state (empty: it does). + ``open_targets`` are the decision's open targets (C-196), whose VERIFIED evidence is a FAIL.""" references = {r.invariant_id: r for r in parent.invariant_refs} problems: list[str] = [] for entry in plan: @@ -238,6 +240,7 @@ def binding_problems( definitions.get(entry.invariant_id), results.get(key), evidence_ids.get(key), + entry.invariant_id in open_targets, ) else: problems += carried_problems( diff --git a/core/promotion/commit.py b/core/promotion/commit.py index 9bb2c8e..a0f245d 100644 --- a/core/promotion/commit.py +++ b/core/promotion/commit.py @@ -463,7 +463,14 @@ def _attempt( definitions = _definitions(uow, plan) results = {r.verification_id: r for r in verification.results} problems = binding_problems( - uow.evidence, plan, candidate, current, definitions, results, evidence_ids + uow.evidence, + plan, + candidate, + current, + definitions, + results, + evidence_ids, + open_targets=outcome.evidence.open_targets, ) if problems: raise _Invalid("the evidence does not bind to the new state: " + "; ".join(problems)) @@ -541,6 +548,7 @@ def _revalidate( now=outcome.decision.created_at, attempt_number=attempt_number, oracle_verdicts=oracle_verdicts, + open_targets=outcome.evidence.open_targets, ) except DomainValidationError as refusal: raise _Invalid(f"decide refuses its inputs now: {refusal}") from refusal @@ -583,7 +591,14 @@ def _build( decision = outcome.decision binding_id = ids.evidence_id_for(decision.decision_id) evaluations = evaluate_from_results( - plan, candidate, current, definitions, results, evidence_ids, binding_id + plan, + candidate, + current, + definitions, + results, + evidence_ids, + binding_id, + open_targets=outcome.evidence.open_targets, ) proofs = build_proofs( plan, candidate, current, impact, evaluations, binding_evidence_id=binding_id diff --git a/core/promotion/decide.py b/core/promotion/decide.py index f8b44df..88a7f59 100644 --- a/core/promotion/decide.py +++ b/core/promotion/decide.py @@ -13,6 +13,16 @@ by severity (C-158), so a later FAIL cannot hide behind an earlier UNKNOWN and the decision does not depend on the order of anything. +**Two extensions for the experiments (P8a), both off by default.** (1) An explicit required set +(``policy.explicit_required_set``, C-205): the required set is the gate items and +``policy.extra_required`` only; an invariant of the current state outside it is an ESCALATE +(POLICY_REGISTRY_REGRESSION), so nothing is dropped. (2) Open targets (``open_targets``, C-196): an +invariant that is VIOLATED in the current state and named as open may FAIL without blocking; a +sufficient fresh FAIL is its VERIFIED evidence and it stays VIOLATED. Anything else about it blocks +as before (a non-success, a missing or insufficient result), every other item is decided as before, +and the open targets are recorded in the decision evidence. ``check_open_targets`` refuses an open +target that is not VIOLATED in the current state or not required. + Standard library and ``core.domain`` only (R18). """ @@ -213,7 +223,8 @@ def _level_ok(record: VerificationResultRecord, policy: PromotionPolicy) -> bool def _required_items( trusted: TrustedState, impact: ImpactReport, policy: PromotionPolicy ) -> tuple[list[_Item], list[BlockingCondition]]: - """Every required target (C-159), and the policy conditions found on the way (C-41).""" + """Every required target (C-159, or C-205's explicit set), and the policy conditions found on + the way (C-41).""" problems: list[BlockingCondition] = [] if policy.invariant_registry_version < trusted.invariant_registry_version: problems.append( @@ -230,7 +241,24 @@ def _required_items( _Item(target, "GATE", None, None, policy.gate_verifier(target)) for target in (GATE_CONSISTENCY, GATE_STRUCTURAL) ] - for invariant_id in sorted({*references, *policy.extra_required}): + if policy.explicit_required_set: + for invariant_id in sorted(set(references) - set(policy.extra_required)): + version = references[invariant_id].invariant_version + problems.append( + _condition( + R.POLICY_REGISTRY_REGRESSION, + D.ESCALATE, + f"INVARIANT:{invariant_id}:{version}", + "the current state holds it but the policy's explicit required set leaves it " + "out; an invariant of the state cannot be dropped (C-41, C-205)", + ) + ) + wanted = ( + set(policy.extra_required) + if policy.explicit_required_set + else {*references, *policy.extra_required} + ) + for invariant_id in sorted(wanted): reference = references.get(invariant_id) definition = policy.definition(invariant_id) if definition is None or ( @@ -258,11 +286,12 @@ def _required_items( (definition.verifier_id, definition.verifier_version), ) ) - items.extend( - _Item(work.target_id, "BASELINE", None, None, None) - for work in impact.verification_work - if work.target_type is WorkTargetType.RESOURCE - ) + if not policy.explicit_required_set: + items.extend( + _Item(work.target_id, "BASELINE", None, None, None) + for work in impact.verification_work + if work.target_type is WorkTargetType.RESOURCE + ) return sorted(items, key=lambda item: item.target_id), problems @@ -328,6 +357,7 @@ def _assess( trusted: TrustedState, impact: ImpactReport, policy: PromotionPolicy, + opened: frozenset[str] = frozenset(), ) -> _Verdict: """Is ``item`` proven, and if not, why not (C-158, C-159)?""" candidate_hash = candidate.state_hash @@ -353,7 +383,10 @@ def _assess( verifier_version=verifier_version, ) if record is not None and is_fresh: - _assess_result(verdict, item, record, verifier_id, verifier_version, candidate_hash, policy) + is_open = invariant is not None and invariant.invariant_id in opened + _assess_result( + verdict, item, record, verifier_id, verifier_version, candidate_hash, policy, is_open + ) return verdict if invariant is not None: refusals = _carry_forward_refusals(item, candidate, trusted, impact, policy) @@ -437,10 +470,17 @@ def _assess_result( verifier_version: str, candidate_hash: str, policy: PromotionPolicy, + is_open: bool = False, ) -> None: """A fresh result decides: FAIL rejects, a non-success follows the table, a sufficient PASS - proves (Doc 09 §11, §18).""" + proves (Doc 09 §11, §18). On an open target (C-196) a sufficient FAIL is the target's VERIFIED + evidence, not a blocking condition; an insufficient one does not prove it.""" target, described = item.target_id, _described(record) + if record.result is VerificationResult.FAIL and is_open: + _assess_open_fail( + verdict, item, record, verifier_id, verifier_version, candidate_hash, policy + ) + return if record.result is VerificationResult.FAIL: verdict.conditions.append(_condition(_failure_code(item), D.REJECT, target, described)) return @@ -493,10 +533,110 @@ def _assess_result( ) +def _proves_violation( + record: VerificationResultRecord, + *, + target_id: str, + candidate_hash: str, + invariant: Invariant, + verifier_id: str, + verifier_version: str, +) -> bool: + """``sufficient`` (Doc 09 §11) with FAIL in place of PASS: the FAIL names this target, this + candidate, this invariant version and the definition's verifier, and carries every field.""" + if record.result is not VerificationResult.FAIL: + return False + if not all( + getattr(record, name) not in (None, "", {}, ()) + for name in ("verifier_id", "verifier_version", "evidence_level", "evidence") + ): + return False + return ( + record.target_id == target_id + and record.candidate_hash == candidate_hash + and record.invariant_id == invariant.invariant_id + and record.invariant_version == invariant.version + and record.verifier_id == verifier_id + and record.verifier_version == verifier_version + ) + + +def _assess_open_fail( + verdict: _Verdict, + item: _Item, + record: VerificationResultRecord, + verifier_id: str, + verifier_version: str, + candidate_hash: str, + policy: PromotionPolicy, +) -> None: + """A FAIL on an open target (C-196): VERIFIED evidence that it stays VIOLATED, when it is + sufficient; otherwise EVIDENCE_NOT_SUFFICIENT, as for a PASS.""" + invariant = item.invariant + assert invariant is not None + if _proves_violation( + record, + target_id=item.target_id, + candidate_hash=candidate_hash, + invariant=invariant, + verifier_id=verifier_id, + verifier_version=verifier_version, + ) and _level_ok(record, policy): + verdict.check = RequiredCheck( + item.target_id, "INVARIANT", ProofSource.VERIFIED, record.verification_id + ) + verdict.result_id = record.verification_id + verdict.entry = ProofPlanEntry( + invariant.invariant_id, + invariant.version, + ProofSource.VERIFIED, + record.verification_id, + None, + (), + ) + return + verdict.conditions.append( + _condition( + R.EVIDENCE_NOT_SUFFICIENT, + D.REJECT, + item.target_id, + f"{_described(record)} is not sufficient evidence for the open target: it must be from " + f"{verifier_id} {verifier_version} at level {policy.min_evidence_level} or above", + ) + ) + + +def check_open_targets( + trusted: TrustedState, policy: PromotionPolicy, open_targets: object +) -> tuple[str, ...]: + """The open targets, sorted, if every one is allowed (C-196); ``PromotionInputError`` if not. + + Each must be an invariant id held VIOLATED by the current state ``trusted`` and required by + ``policy``. The empty set is always allowed.""" + if isinstance(open_targets, str) or not isinstance(open_targets, Iterable): + raise PromotionInputError("open_targets: must be a list of invariant ids") + names = list(open_targets) + if not all(isinstance(n, str) and n for n in names) or len(set(names)) != len(names): + raise PromotionInputError("open_targets: invariant ids, each once") + references = {ref.invariant_id: ref for ref in trusted.invariant_refs} + for name in names: + reference = references.get(name) + if reference is None or reference.status is not InvariantStatus.VIOLATED: + shown = "absent" if reference is None else reference.status.value + raise PromotionInputError( + f"open_targets: {name} is {shown} in the current state; an open target must be " + "VIOLATED there (C-196)" + ) + if policy.explicit_required_set and name not in policy.extra_required: + raise PromotionInputError(f"open_targets: {name} is not in the policy's required set") + return tuple(sorted(names)) + + def _oracle_conflicts( verdicts: Iterable[OracleVerdict], results: dict[str, _Verdict], policy: PromotionPolicy, + opened: frozenset[str] = frozenset(), ) -> list[BlockingCondition]: """Primary PASS (or a carried proof) and oracle FAIL: a conflict, and the check is not proven (C-160). Every other pairing leaves the primary result as it is.""" @@ -504,6 +644,8 @@ def _oracle_conflicts( for oracle in sorted(verdicts, key=lambda v: (v.invariant_id, v.invariant_version)): if oracle.result is not VerificationResult.FAIL: continue + if oracle.invariant_id in opened: # the primary says FAIL too: no conflict (C-196) + continue target = f"INVARIANT:{oracle.invariant_id}:{oracle.invariant_version}" verdict = results.get(target) if verdict is None or verdict.check.source is None: @@ -569,12 +711,14 @@ def decide( now: datetime, attempt_number: int = 1, oracle_verdicts: Iterable[OracleVerdict] = (), + open_targets: Iterable[str] = (), ) -> DecisionOutcome: """PROMOTE, REJECT, RETRY or ESCALATE for ``candidate`` against the current state ``trusted``. ``attempt_number`` counts the attempts at this candidate's problem (1 for the first); it feeds only the retry budget of the policy. PROMOTE needs every required item proven, nothing - conflicting, a complete decision and the emergency switch on (Doc 09 §19).""" + conflicting, a complete decision and the emergency switch on (Doc 09 §19). ``open_targets`` + (C-196, default none) are checked by ``check_open_targets`` and recorded in the evidence.""" moment, verdicts = _check_arguments( candidate, trusted, @@ -586,6 +730,7 @@ def decide( attempt_number, oracle_verdicts, ) + opened = check_open_targets(trusted, policy, open_targets) if candidate.parent_state_id != trusted.state_id: stale = _condition( R.STALE_PARENT, @@ -594,18 +739,22 @@ def decide( f"the candidate's parent {candidate.parent_state_id} is not the current state " f"{trusted.state_id}", ) - return _build(candidate, policy, decision_id, moment, attempt_number, [], [stale], (), ()) + return _build( + candidate, policy, decision_id, moment, attempt_number, [], [stale], (), (), opened + ) items, conditions = _required_items(trusted, impact, policy) conditions += _binding_conditions(candidate, impact, verification) by_id: dict[str, list[VerificationResultRecord]] = {} for result in verification.results: by_id.setdefault(result.target_id, []).append(result) + open_set = frozenset(opened) assessed = { - item.target_id: _assess(item, by_id, candidate, trusted, impact, policy) for item in items + item.target_id: _assess(item, by_id, candidate, trusted, impact, policy, open_set) + for item in items } for verdict in assessed.values(): conditions += verdict.conditions - conditions += _oracle_conflicts(verdicts, assessed, policy) + conditions += _oracle_conflicts(verdicts, assessed, policy, open_set) ordered = [assessed[item.target_id] for item in items] checks = [v.check for v in ordered] plan = sorted((v.entry for v in ordered if v.entry is not None), key=lambda e: e.invariant_id) @@ -645,7 +794,16 @@ def decide( _condition(R.DECISION_INCOMPLETE, decision, "DECISION", "; ".join(problems)) ) return _build( - candidate, policy, decision_id, moment, attempt_number, checks, conditions, plan, supporting + candidate, + policy, + decision_id, + moment, + attempt_number, + checks, + conditions, + plan, + supporting, + opened, ) @@ -659,6 +817,7 @@ def _build( conditions: Sequence[BlockingCondition], plan: Sequence[ProofPlanEntry], supporting: Sequence[str], + opened: Sequence[str] = (), ) -> DecisionOutcome: unique = sorted({c.sort_key(): c for c in conditions}.values(), key=BlockingCondition.sort_key) if unique: @@ -690,5 +849,6 @@ def _build( policy_version=policy.policy_version, decision=decision, reason=reason, + open_targets=opened, ) return DecisionOutcome(made, tuple(plan), evidence) diff --git a/core/promotion/policy.py b/core/promotion/policy.py index 9ef9b0a..048f6bf 100644 --- a/core/promotion/policy.py +++ b/core/promotion/policy.py @@ -1,10 +1,15 @@ -"""The two promotion policies (P6a; Doc 09 §20, §28; C-22, C-157 to C-161). +"""The promotion policies (P6a; Doc 09 §20, §28; C-22, C-157 to C-161; P8a: C-198, C-205). ``dail_policy`` is the DAIL condition (carry-forward on). ``stateful_full_policy`` is the Stateful-Full comparison condition of Doc 13: the same controller, the same table, the same budget, with carry-forward disabled, so every required invariant has to be verified again (C-22). The two differ in one field and a test pins that. +``stateless_policy`` (P8a) is condition A, Stateless: Stateful Full with an explicit required set +(``explicit_required_set``): the gate items and the ``required`` invariants only, no baseline item +and no other invariant of the state (C-205). Its rules carry the version +``STATELESS_RULES_VERSION``. + Nothing here reads configuration. ``automatic_enabled`` has no default: the caller passes the reviewed ``promotion.automatic_enabled`` value (Doc 14 §37), so a policy can never be built with the emergency switch left to chance. @@ -34,6 +39,8 @@ ) DEFAULT_MIN_EVIDENCE_LEVEL: Final = "L1" DEFAULT_MAX_ATTEMPTS: Final = 3 +# The version of the rules ``stateless_policy`` builds (C-205); a change to them is a new version. +STATELESS_RULES_VERSION: Final = "dail-stateless-1" def _policy( @@ -48,6 +55,7 @@ def _policy( max_attempts: int, conflict_decision: DecisionType, min_evidence_level: str, + explicit_required_set: bool = False, ) -> PromotionPolicy: return PromotionPolicy( policy_version=policy_version, @@ -63,6 +71,7 @@ def _policy( automatic_enabled=automatic_enabled, accept_oracle_for_unknown=False, gate_verifiers=GATE_VERIFIERS, + explicit_required_set=explicit_required_set, ) @@ -118,3 +127,33 @@ def stateful_full_policy( conflict_decision=conflict_decision, min_evidence_level=min_evidence_level, ) + + +def stateless_policy( + *, + policy_version: str, + invariants: Iterable[Invariant], + automatic_enabled: bool, + required: Iterable[str], + invariant_registry_version: int = 1, + transient_errors: Iterable[str] = (), + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + conflict_decision: DecisionType = DecisionType.ESCALATE, + min_evidence_level: str = DEFAULT_MIN_EVIDENCE_LEVEL, +) -> PromotionPolicy: + """Condition A, Stateless (C-198, C-205): the gate items and ``required`` only, nothing is + carried forward. ``required`` is the repair target, or nothing for a change request (C-197); + every id must be defined in ``invariants``. Rules version ``STATELESS_RULES_VERSION``.""" + return _policy( + carry_forward_enabled=False, + policy_version=policy_version, + invariants=invariants, + automatic_enabled=automatic_enabled, + invariant_registry_version=invariant_registry_version, + extra_required=required, + transient_errors=transient_errors, + max_attempts=max_attempts, + conflict_decision=conflict_decision, + min_evidence_level=min_evidence_level, + explicit_required_set=True, + ) diff --git a/core/promotion/proofs.py b/core/promotion/proofs.py index 25b5687..deef65c 100644 --- a/core/promotion/proofs.py +++ b/core/promotion/proofs.py @@ -14,12 +14,14 @@ Every proof also carries the id of the promotion binding record (C-164). A plan entry without an evaluation or an evidence id, a result that is not a PASS, a version that differs from the parent's reference (the lifecycle has no path for a version change) and a plan that leaves out an invariant -of the parent are refused: fail closed. Standard library and ``core.domain`` only (R18). +of the parent are refused: fail closed. The one exception is an open target (C-196, P8a): its +VERIFIED entry names a FAIL, and its evaluation ends VIOLATED; any other FAIL is still refused. +Standard library and ``core.domain`` only (R18). """ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from core.domain.enums import VerificationResult from core.domain.errors import DomainValidationError @@ -53,6 +55,7 @@ def evaluate_from_results( results: Mapping[str, VerificationResultRecord], evidence_ids: Mapping[str, str], binding_evidence_id: str, + open_targets: Collection[str] = (), ) -> dict[str, InvariantEvaluation]: """The evaluation of every VERIFIED entry of ``plan``, by invariant id. @@ -75,7 +78,12 @@ def evaluate_from_results( f"proof plan: {entry.target_id} names result {entry.verification_id}, which has no " "stored result or no evidence record" ) - if record.result is not VerificationResult.PASS: + allowed = ( + (VerificationResult.PASS, VerificationResult.FAIL) + if entry.invariant_id in open_targets + else (VerificationResult.PASS,) + ) + if record.result not in allowed: raise DomainValidationError( f"proof plan: {entry.target_id} is VERIFIED by a {record.result.value} result" ) diff --git a/docs/BUILD_SEQUENCE.md b/docs/BUILD_SEQUENCE.md index 2d3d463..0444226 100644 --- a/docs/BUILD_SEQUENCE.md +++ b/docs/BUILD_SEQUENCE.md @@ -31,7 +31,7 @@ The build is split into 18 work packages (N-06 in `DECISIONS_REGISTER.md`). Ever | 19 | Dec 1–2 | P7a LLM adapter (fake provider, Ollama) | | 20 | Dec 3–6 (built 12 Nov 2026) | P7b patch validation: the C-188 patch meaning, the §13 validator, AllowedChangeScope and the operation check, `LLMEvaluationRequest` (C-188 to C-195) → P7 gate, M4 | | 21 | — | ~~P9 AWS integration (free route) → M6~~ removed by C-71 (advisor, 11 Nov 2026): AWS stays fully offline, no deployment, no AWS account, no live calibration; milestone M6 (AWS Demonstration) is removed | -| 22 | Dec 12–14 | P8a experiment harness | +| 22 | Dec 12–14 (built 12 Nov 2026) | P8a experiment harness: conditions A, B, C, C′ (C-198), open targets (C-196), the TerraPreserve loader with hash pins and label blinding, trials, records and artifacts, the dev-split harness check (C-196 to C-208) | | 23 | Dec 15–18 | P8b metrics and reports → P8 gate, M5 | | 24–28 | Dec 19–Jan 2 | P10 hardening, protocol freeze + pilot, LLM patch generation, main run, QA report | | 29 | Jan 3–4 | TerraPreserve v1.1 (real LLM patches) | diff --git a/docs/DECISIONS_REGISTER.md b/docs/DECISIONS_REGISTER.md index dcdf791..99ff585 100644 --- a/docs/DECISIONS_REGISTER.md +++ b/docs/DECISIONS_REGISTER.md @@ -31,7 +31,7 @@ This register records every conflict found between the 15 `CloudSpartanX_*` spec | N-02 | Oracle separation | Separate repository and different author, written from AWS semantics and the registry spec (ADR-010) | Accepted | | N-03 | Dataset families F3/F7 | Route edits cannot break intra-VPC reachability (the VPC local route cannot be deleted). F3 unsafe = move EC2 to a subnet outside a CIDR-based RDS ingress rule; F7 unsafe = edit a shared hub security group; route edits stay as safe cases | Resolved by C-69 (step 4) | | N-04 | Impact ground truth | Two sets. `truth_changed`: the oracle's value differs from the previous step, and DAIL recall on it must be 100%. `relevant`: a changed attribute lies inside the invariant's footprint, and precision/recall are measured here | Resolved by C-70 (step 4) | -| N-05 | Condition A on change-request families | For F1, F5 and F7, Stateless runs structural checks only; declared in the frozen protocol | Pending protocol freeze | +| N-05 | Condition A on change-request families | For F1, F5 and F7, Stateless runs structural checks only; declared in the frozen protocol. **Amended by C-197 (authors, 12 Nov 2026):** the rule applies to every scenario whose `target.kind` is `change_request` (in TerraPreserve v1.0: F1, F3, F5, F6, F7), not to a list of families. | Pending protocol freeze | | N-06 | Guide granularity | 18 work packages (`BUILD_SEQUENCE.md`) | Accepted | | N-07 | Safety-flag enabling | Safety-critical flags can be set to the conservative value from any configuration layer except runtime override, and enabled only in a reviewed configuration file (interpretation of Doc 14 §15, §37; amended by C-26) | Accepted | | C-24 | Dev-tool pins vs dependency scan | black 26.3.1 and pytest 9.0.3 replace black 24.10.0 and pytest 8.3.3 so that `pip-audit` reports no known vulnerabilities (PYSEC-2026-2120, PYSEC-2026-2121, PYSEC-2026-1845); `requirements/dev.lock` regenerated with pip-compile (Doc 14 §23, §39) | Accepted 30 Sep 2026 | @@ -207,3 +207,19 @@ This register records every conflict found between the 15 `CloudSpartanX_*` spec | C-193 | P7b-5: prompt version | The prompt must now describe the patch format of C-188, so it gets a new version `dail-repair-prompt-2` with a new pinned hash; `dail-repair-prompt-1` stays reproducible. | Proposed (P7b) | | C-194 | P7b: `llm.api_key_ref` stays in the schema | The work package asked to remove the unused key and its pinned-key test entry. It is the only `SECRET_REF` key of the schema: about ten tests in `tests/unit/test_config.py` and `tests/unit/test_health.py` exercise the `env:NAME` secret machinery and the C-25 reserved-prefix rule through it, so removing it would delete or rewrite them (CLAUDE.md: tests only get stronger). **User's decision (12 Nov 2026): keep the key, record why.** No provider reads it (C-71: no keys). | Decided (user, 12 Nov 2026) | | C-195 | P7b: readings made while building the patch validator | (1) **Delete-only patch.** The parser keeps P7a's rule that a PATCH needs a non-empty patch string, so a patch that only deletes holds a comment line; prompt 2 says so. (2) **Attributes (C-190).** A non-empty `attributes` restricts MODIFY only: CREATE has no parent block to compare with and DELETE has no patch block. Arguments and nested blocks are compared by name, each with its whitespace-normalized source texts. (3) **Lifecycle.** Compared as the list of whitespace-normalized `lifecycle` blocks of the resource; a CREATE has no parent, so any `lifecycle` in it is POLICY_VIOLATION. (4) **Credentials (§14).** A secret-named argument (the `Redactor` key rule) at any depth, identified by its path of block types and name, must have the whitespace-normalized text of the parent's argument at that path or be a `var.` reference. (5) **Invented ids (§14).** The id pattern is matched inside every string literal and template text, bounded by non-alphanumeric characters; an `arn:aws:` string counts as invented when it does not occur in the parent text. (6) **References.** The roots `count`, `each`, `self`, `path`, `terraform` and for-expression iterators are not configuration names; P3's loader would also refuse an undeclared reference later, so check 9 is a first line, and the tests require that check 9 itself reports. (7) **Fail closed.** MODIFY or DELETE of a block that lives in a `.tf.json` file is UNSUPPORTED; a parent that already has `dail_llm_patch.tf` is UNSUPPORTED for CREATE; a parser limit (C-93) is UNSUPPORTED; a built candidate whose normalization is UNSUPPORTED is UNSUPPORTED and no candidate exists (Doc 03 §4.7 lets a pipeline continue only where it permits; P7b does not). (8) **Exact parent.** The validator refuses, at construction, a parent directory whose normalized resource set does not have the trusted state's `resource_set_hash` (Doc 03 §4.5 step 1). (9) **Operation check (C-191).** It runs inside the validator with DAIL's own identity engine against the trusted state's resources; the orchestrator runs identity again on the same inputs, so the result is the same. (10) **Entry path.** `LLMEvaluationRequest` is a subclass of `EvaluationRequest` (one evaluation path) that accepts only an LLM `Patch`, carries `built_tree_hash`, and recomputes the directory's tree hash; a fixed-patch request still refuses an LLM patch. (11) **Context and validator agree.** `Generator` refuses a context whose parent state or change scope is not the validator's. (12) **Builds.** A refused build is removed at once; a successful one stays under the caller's `work_root`, which the caller owns. | Proposed (P7b) | +| C-196 | P8a: open repair target on the middle steps of a multi-step repair | In a TerraPreserve scenario with more than one step after S0 (family F8), the harness marks the scenario's repair target as an *open target* for every step except the last. For an open target, a FAIL result on the candidate does not block promotion, and the invariant stays VIOLATED in the new trusted state with the fresh FAIL evidence. Every other required invariant is decided exactly as before. An open target must be VIOLATED in the parent (the request is refused otherwise), so the rule can never hide a regression of a PROTECTED invariant. It still needs a fresh verification result: UNKNOWN, UNSUPPORTED or VERIFIER_ERROR on it blocks as before, and a missing result blocks. The last step has no open target, so the target must PASS there. The rule is the same for every condition. The default is no open target, so every existing behaviour, including Doc 15 §13.2 "SEC FAIL + FUNC PASS → REJECT", is unchanged. The open targets of each transition are recorded in its decision evidence. Reason: TerraPreserve's labels (D5-11) let a repair target stay FAIL until the last step; without this rule every condition rejects F8 at step 1 and F8 measures nothing. | Decided (authors, 12 Nov 2026) | +| C-197 | P8a: N-05 amended | Condition A's rule for scenarios without a repair target applies to **every scenario whose `target.kind` is `change_request`** (in v1.0: F1, F3, F5, F6, F7), not to a list of families. N-05 points here. | Decided (authors, 12 Nov 2026) | +| C-198 | P8a: the conditions for P8 (C-20, C-22, N-05) | A, B, C and C′. All four use the same verifiers, the same orchestrator and the same promotion controller (RV-001); they differ only in this: **A, Stateless:** evidence identity; verification work = the gate items (STRUCTURAL, CONSISTENCY) and the repair target's invariant item, or the gate items only for `change_request` scenarios (C-197); required set = the gate items and the repair target only; no ledger use, no carry-forward. **B, Stateful Full:** evidence identity; verification work = the gate items, every invariant on the current state and every RESOURCE baseline item of the impact work set; required set as `stateful_full_policy` (every invariant on the state, carry-forward off). **C, DAIL:** evidence identity; `DailScopePolicy` unchanged; `dail_policy` (carry-forward on). **C′, DAIL without identity evidence (H3, C-20):** `AddressOnlyIdentity` (C-127), `DailScopePolicy`, as C. | Decided (authors, 12 Nov 2026) | +| C-199 | P8a: P7a, P7b, P7 gate and M4 status | The user pasted the verdicts on PR #28 and PR #29 (C-28). `docs/PHASE_GATES.md`: "P7a: passed with limits (verdict on PR #28)", "P7b: passed with limits (verdict on PR #29)", "P7 gate (Doc 15 §14.2) and milestone M4 (Doc 15 §24): passed with limits (verdict on PR #29)"; the limits of both verdicts are listed there as open items with the package that closes each (P8 or P10). | Decided (authors, 12 Nov 2026) | +| C-200 | P8a-1: trial unit and trajectory | A trial is one TerraPreserve target scenario under one condition (repetition 1; fixed patches are deterministic). Its trajectory evaluates steps 1..n in order, each as a FIXED_PATCH candidate against that trial's current trusted state. After any decision other than PROMOTE at step i < n, steps i+1..n are not evaluated and are recorded as TRUNCATED (early termination; the paper reports it and runs a common-prefix sensitivity analysis in P8b). | Proposed (P8a) | +| C-201 | P8a-2: admission (C-99) | A TerraPreserve step enters DAIL only if its recomputed `content_hash` equals the frozen manifest's value; every v1.0 step passed `terraform validate` 1.15.8 at D6 gate 2, and that is recorded as its admission evidence. DAIL does not run Terraform in P8a. | Proposed (P8a) | +| C-202 | P8a-3: isolation | Each trial has its own fresh store (its own SQLite file and artifact directory) and its own lineage id. Nothing is shared between trials or conditions except the read-only scenario files. No experiment lineage is ever the authoritative lineage (Doc 13 §20). | Proposed (P8a) | +| C-203 | P8a-4: registry | Every condition uses invariant registry **v1.1** (INV-SEC-001/002 version 2). TerraPreserve's `protected_invariants` (registry v1.0 ids) map to the v1.1 definitions; P6d showed the v1.0 labels are identical under v1.1. Every record that carries an invariant version also carries the mapping "stored invariant_registry_version 2 = registry 1.1" (C-180). | Proposed (P8a) | +| C-204 | P8a-5: where the run lives | Run output goes under `$DAIL_EXPERIMENT_ROOT` (outside the repository, never committed). The TerraPreserve root comes from `$TERRAPRESERVE_ROOT`. | Proposed (P8a) | +| C-205 | P8a: condition A's required set and C-41 | The work package asks for a policy that requires only the gate items and the repair target, but C-40/C-41 forbid a promotion from dropping an invariant of the current state (`build_proofs`). **User's decision (12 Nov 2026): A's lineage tracks only what A requires.** A's baseline registers only the repair target's definition (and only the bindings of that invariant); the policy constructor `stateless_policy` sets `explicit_required_set = True`, so the required set is exactly the gate items and `extra_required` (no RESOURCE baseline items, no other invariant of the state), with carry-forward off. A current state that holds an invariant outside the explicit set is an ESCALATE (`POLICY_REGISTRY_REGRESSION`, C-41), so nothing is dropped silently. C-41 is unchanged. A's S0 records cover only that set. | Decided (user, 12 Nov 2026) | +| C-206 | P8a: condition A on `change_request` scenarios needs a baseline with no invariant | `TrustedState.establish_baseline` refuses a baseline with no invariant proof (cited as Doc 06 §24), and A tracks no invariant for a `change_request` scenario (C-197, C-205). **User's decision (12 Nov 2026): an explicit opt-in.** `TrustedState.establish_baseline(..., allow_no_invariants=False)` and `BaselineRequest.allow_no_invariants = False`: only with the flag may the set be empty; every existing caller keeps the "at least one" rule. Only the experiment harness sets it, for A on `change_request`. Recorded as a reading of Doc 06 §24 for the reviewer: the spec says the baseline establishes "its protected invariants" and does not say the set is non-empty. | Decided (user, 12 Nov 2026) | +| C-207 | P8a: readings made while building the harness | (1) **Condition items.** A work item that the condition's scope policy adds for an invariant impact did not list (B's unaffected invariants; A's target when impact did not list it) needs a `ReasonCode`, and C-138's vocabulary is closed (a test pins it) with no code for "the condition re-verifies it"; the stand-in is `SCOPE_TYPE_CHANGED` (`scope_policy.CONDITION_ITEM_REASON`). Nothing reads a work item's reason after selection (the verification request does not carry it); the records mark such items `origin = condition`. Its scope is the definition's scope (resources, else resource types); the verifiers do not read it. Question for the reviewer: a new reason code through a C-138 row instead? (2) **Registry history.** A fresh store must hold version 1 of INV-SEC-001/002 before version 2 (Doc 06 §4.2, no gaps), so each trial store registers the registry-v1.0 definitions of the ids it tracks first; the baseline registers v1.1. (3) **Automatic promotion.** `tests/unit/test_config.py::test_real_config_loads_for_all_four_environments` pins `promotion.automatic_enabled = false` in every environment of `config/`, so the harness does not change `config/`: it loads a copy of `config/` with `base/promotion.toml` replaced by the reviewed file `experiments/config/promotion.toml` (`automatic_enabled = true`, a versioned file, never an environment variable or override), environment `experiment`. Its SHA-256 is pinned in every definition. Without it every PROMOTE would be an ESCALATE. (4) **Layout.** §41's `trials//{baseline/ ...}` gets an `attempts//` level, so that a re-run never overwrites the original (§47); each attempt has its own `store/dail.sqlite`. The lifecycle is `trials//events.jsonl` (append-only). (5) **Time.** Everything DAIL hashes uses a logical clock (2026-01-01T00:00:00Z plus one second per call), so a re-run stores the same records; wall-clock times and durations are only in `timing.json` and the events. (6) **Random ids.** The identity and impact reports draw a random `report_id` (the orchestrator does not pass one); the stored copies leave it out and keep the content hash. (7) **Outcomes.** A construction failure is recorded as a REJECT with reason `CONSTRUCTION_FAILED` (DAIL stores no decision, C-170 (3)). A baseline that DAIL refuses (C-171) leaves the trial COMPLETED with every step TRUNCATED. An open target that DAIL's own parent state does not hold VIOLATED is refused by `check_open_targets` (C-196) and the trial is FAILED as a HARNESS_FAILURE. A stale decision records no open target. A FAILED trial keeps its baseline artifacts and no transition in its result. (8) **Lifecycle (§7).** A trial interrupted before it executed may go INITIALIZED → ABORTED; an ABORTED trial is re-run as a new attempt (ABORTED → INITIALIZED); a FAILED trial is not re-run. (9) **Loader.** The manifest is parsed whole (its bytes are pinned) but only the `dev` list is indexed; `template_id` is passed (not the source lineage id); the admission evidence names D6 gate 2 only for the pinned v1.0 manifest. (10) **Conditions.** B's "every invariant on the current state" is every definition passed (B's lineage holds all four); A evaluates with the four definitions (graph, impact, verification inputs) but tracks and verifies only its required set. A baseline `state_hash` covers the lineage (C-33), so S0 consistency across conditions is checked on `resource_set_hash`. | Proposed (P8a) | +| C-208 | P8a: the identity-strategy injection point and the P3b test that forbade it | The work package asks for the narrowest injection point in the application layer for condition C′ (`AddressOnlyIdentity`, C-127). `tests/unit/test_identity_strategy.py::test_no_configuration_key_or_environment_variable_selects_a_strategy` rejected any mention of `identity_strategy`, `ablation` or `address-only` in `core/application/*.py`. **User's decision (12 Nov 2026): keep the injection point and amend the test, keeping its intent.** `EvaluationOrchestrator(..., identity_strategy=DEFAULT_STRATEGY)`; `config/` and `scripts/` (and `core/application/config.py`) are still scanned for those words exactly as before; the rest of `core/application` is checked by a new test, `test_the_application_layer_reads_no_strategy_switch`: no module names `AddressOnlyIdentity` or `ADDRESS_ONLY`, none reads `os.environ` or `getenv` (the configuration loader excepted), no `get(...)` or `[...]` lookup uses a key that names an identity, a strategy or the ablation, and the orchestrator's default is `EvidenceIdentity`. Two mutants (a strategy read from a configuration key, and from the environment) are caught. Only `experiments/` names the ablation. | Decided (user, 12 Nov 2026) | +| C-209 | P8a dev run: under C and C′ the open target of C-196 rejects every F8 trajectory at step 1 | Found in the P8a harness check (no label read). C-196 says an open target "still needs a fresh verification result ... a missing result blocks", and its reason is that "without this rule every condition rejects F8 at step 1". In the dev run, on all 28 F8 targets, C and C′ REJECT step 1 with `INSUFFICIENT_EVIDENCE` on `INVARIANT:INV-SEC-001:2`: the middle step does not touch INV-SEC-001's footprint, so impact does not list it, `DailScopePolicy` (unchanged, C-198) owes no work item for it, and a VIOLATED reference cannot be carried forward (C-159), so no fresh result exists and the rule blocks. A and B verify the target on every step and promote these steps (A 84 of 84 F8 transitions, B 63 of 77). So for C and C′ the rule does not reach its stated purpose, and F8 measures only step 1 there. Nothing was changed (no tuning to the dev run). Options for the authors include: let an open target that the candidate does not affect stay VIOLATED with the parent's evidence (a carry of a VIOLATED reference, which the commit path already supports through `InvariantProof.carry_forward`); or make the harness add the open target's item to every condition's work set. The same mechanism also rejects a candidate whose parent holds a non-target invariant VIOLATED that the candidate does not touch (1 transition each under C and C′). **Closed by C-210 (authors, 12 Nov 2026).** | Decided (authors, 12 Nov 2026): see C-210 | +| C-210 | P8a-2: DAIL always re-checks every invariant that is not PROTECTED in the parent (closes C-209) | DAIL's verification work for a candidate is the impact work set plus one invariant item for every invariant whose reference in the parent trusted state is not PROTECTED (VIOLATED or UNCERTAIN) and that impact did not already list. This is the paper's required set R(C_t) = G_gate ∪ U_t ∪ I_t ∪ B, where U_t is the set of non-PROTECTED references (C-159 already requires a fresh result for each of them; the work set did not schedule one). It applies to conditions C and C′ (both use `DailScopePolicy`). These items count as verification work. A VIOLATED reference is never carried forward (C-159 unchanged). **Built:** `DailScopePolicy` version `dail-scope-2` adds the U_t items (reason `NOT_PROTECTED_IN_PARENT`, BLOCKING, sorted, never a second item for an invariant impact listed). `VerificationScopePolicy.select` takes the parent state's invariant references as a keyword `parent_refs` (default none): the narrowest input that carries the statuses, which the orchestrator already holds; the impact report does not carry them and a scope policy does not read the database. Impact, verification, identity and the promotion decision are unchanged; with every parent reference PROTECTED the work set is impact's, unchanged. | Decided (authors, 12 Nov 2026) | +| C-211 | P8a-2: two reason codes for work a scope policy adds (amends C-138's vocabulary; answers C-207 (1)) | `ReasonCode.CONDITION_SCOPE` (the condition's scope policy requires this check: B's unaffected invariants, A's target when impact did not list it) and `ReasonCode.NOT_PROTECTED_IN_PARENT` (the U_t items of C-210), in a fifth group `POLICY_CODES`. They are valid only on invariant work items, and an impact report that carries one is refused (impact never gives them). The `SCOPE_TYPE_CHANGED` stand-in of C-207 (1) is gone; the experiment records keep `origin = condition` and add the item's reason (`selected_reason`). `tests/unit/test_impact_domain.py` lists the two new codes and the new group explicitly (stricter). `StatefulFullScopePolicy` and `StatelessScopePolicy` are now version 2 (their items' reason changed). | Decided (authors, 12 Nov 2026) | diff --git a/docs/PHASE_GATES.md b/docs/PHASE_GATES.md index 54c10b0..12a471d 100644 --- a/docs/PHASE_GATES.md +++ b/docs/PHASE_GATES.md @@ -303,7 +303,27 @@ v1.1 by tests.unit.test_registry_v1_1. Carry-forward: C-181 Proposed; the user t after the verdict and the merge. ``` -## P7a — The LLM channel (pending external review) +## P7a — The LLM channel — passed with limits (verdict on PR #28) + +```text +P7a: passed with limits (verdict on PR #28). Recorded in P8a (C-199); limits recorded in P8a-2. +Limits of the verdict (open items, with the package that closes each): +1. C-183 to C-187 Proposed (now Decided in P7b). Closed. +2. Provider interface differs from Doc 10 §30 (ADR-006): state as a deviation in the paper. + Open (paper). +3. Scope check refused new-address repairs. Closed in P7b (C-190). +4. Ollama 400/404 map to UNSUPPORTED, not retried: P10 counts them separately from model failures. + Open (P10). +5. Token budget is an estimate (ceil(characters / 4)): P10 reports it as an estimate and checks the + model's real context window. Open (P10). +6. The wiring check proves the plumbing only; model and licence checked by the author. Open (P10 + model choice). +7. Weak red run, compensated by mutations. Closed. +8. Remove llm.api_key_ref (kept by C-194) and the Pydantic line in CLAUDE.md (done in P7b). Closed. +9. Ollama left running on the author's machine; the installer can be deleted. Open (user). +``` + +The P7a author's input to the review, as written before the verdict: ```text P7a mini-gate: NOT DECIDED. Status written by the P7a author as an input to the external review. @@ -333,7 +353,34 @@ loopback fake server, not the real one; the wiring check ran once on one machine Carry-forward: C-183 to C-187 Proposed; ADR-006 Proposed. ``` -## P7b — From LLM patch to Candidate State (pending external review) +## P7b — From LLM patch to Candidate State — passed with limits (verdict on PR #29) + +```text +P7b: passed with limits (verdict on PR #29). Recorded in P8a (C-199); limits recorded in P8a-2. +Limits of the verdict (open items, with the package that closes each): +1. C-189 to C-193 and C-195 Proposed: the authors decide. Open. +2. Untested branch: a non-var. reference as a credential value (for example password = local.x -> + POLICY_VIOLATION) needs a test. Open (add it in this fix round, see Step 4). [Test added in + P8a-2: tests.unit.test_llm_patch_validator.TestCredentialsAndIds + .test_a_secret_argument_from_a_local_value_is_a_policy_violation.] +3. The invented-id check sees whole string literals only (an id split inside a template passes). + Open (stated limit). +4. A MODIFY may drop the parent's lifecycle block. 14 of the 288 dev scenarios have a lifecycle + block in S0. Open (P10 decides). +5. The tree hash is checked when the request is built, not at evaluation time: P10 builds and + evaluates each LLM candidate in one step and deletes the build. Open (P10). +6. The candidate directory copies every parent file: P10 passes configuration-only parents. + Open (P10). +7. Experiment scope for LLM candidates not fixed yet: one scope per scenario, the same for every + condition, recorded in advance. Open (P10). +8. The wiring run proves the path only (the 1.5B model kept public SSH; DAIL rejected it). Open + (P10 model choice). +9. Delete-only patches need a comment as patch text: P10 reports how often models fail on this. + Open (P10). +10. llm.api_key_ref stays (C-194). Open (unused). +``` + +The P7b author's input to the review, as written before the verdict: ```text P7b mini-gate: NOT DECIDED. Status written by the P7b author as an input to the external review. @@ -366,7 +413,14 @@ Carry-forward: C-189 to C-193 and C-195 Proposed; ADR-006 Proposed. The P7a mini is unchanged: no verdict text for P7a was pasted in this package. ``` -## P7 — LLM Integration (pending external review) +## P7 — LLM Integration — passed with limits (verdict on PR #29) + +```text +P7 gate (Doc 15 §14.2) and milestone M4 (Doc 15 §24): passed with limits (verdict on PR #29). +Recorded in P8a (C-199). The limits are those of the P7b verdict, listed in the P7b section. +``` + +The gate map written by the P7b author as an input to the review: ```text P7 gate (Doc 15 §14.2) and milestone M4 (Doc 15 §24): NOT DECIDED. Each item and the tests that @@ -408,7 +462,72 @@ M4 "No model output can bypass promotion" ..... test_llm_candidate_e2e.TestSecur No item is marked passed here; the external review decides. ``` -## P8–P10 — NOT STARTED +## P8a — The experiment harness (pending external review) + +```text +P8a mini-gate: NOT DECIDED. Status written by the P8a author as an input to the external review. +What exists: + experiments/ (standard library, mypy-checked): loader (TerraPreserve read only; manifest pin + 665b326a..., every scenario_json_sha256 and step content_hash recomputed; the test split refused; + execution fields only), definition (Experiment §5, Trial §6, lifecycle §7, deterministic ids), + conditions (C-198), trial (one attempt: fresh store and lineage, S0 through the baseline flow, + each step a FIXED_PATCH through EvaluationOrchestrator, early termination C-200), records, + write-once artifacts (§41 plus an attempts level), runner (plan, run, resume, failure classes), + CLI python -m experiments run|summary. README in experiments/. + core/promotion: decide(open_targets=...) and check_open_targets (C-196); explicit required set + (PromotionPolicy.explicit_required_set, stateless_policy, rules dail-stateless-1, C-205); the + open target's FAIL is VERIFIED evidence through proofs, binding and commit; PromotionEvidence + records open_targets (absent from the record when empty, so existing hashes are unchanged). + core/application: StatefulFullScopePolicy and StatelessScopePolicy; EvaluationRequest.open_targets; + EvaluationOrchestrator(identity_strategy=DEFAULT_STRATEGY) (C-208); EvaluationResult.identity and + .work; BaselineRequest.allow_no_invariants (C-206). core/domain: TrustedState.establish_baseline + (allow_no_invariants=False) (C-206). +Dev run (12 Nov 2026, the author's machine): experiment p8a-dev-harness-check version 1, + protocol_version draft, id fdbef60c-9b45-53dd-8e6a-5c6b765505cc; DAIL e8f43c3 (clean tree); + TerraPreserve terrapreserve-v1.0 (27f0ee4), manifest sha256 665b326a... verified; registry v1.1 + (stored version 2); Python 3.12.10, Windows 11. 224 dev targets x A, B, C, C' = 896 trials, + 896 COMPLETED, 0 FAILED, 0 harness or input failures; wall time 903 s. + Transitions evaluated / TRUNCATED: A 280 / 0, B 273 / 7, C 224 / 56, C' 224 / 56. + Transitions with an open target: A 56, B 56, C 28, C' 28. + Work (gate, invariant, applicable baseline, not-applicable baseline): A 560, 140, 0, 0; + B 546, 1092, 56, 10; C 448, 510, 56, 10; C' 448, 510, 62, 16. + Decisions per condition and family (PROMOTE/REJECT; no RETRY or ESCALATE occurred): + A: F1 28/0, F2 14/14, F3 28/0, F4 28/0, F5 28/0, F6 28/0, F7 28/0, F8 84/0. + B: F1 14/14, F2 14/14, F3 11/17, F4 14/14, F5 13/15, F6 9/19, F7 14/14, F8 63/14. + C: F1 14/14, F2 14/14, F3 11/17, F4 14/14, F5 13/15, F6 9/19, F7 14/14, F8 0/28. + C': F1 14/14, F2 14/14, F3 11/17, F4 14/14, F5 9/19, F6 9/19, F7 14/14, F8 0/28. + These are harness counts, not results; no label was opened and nothing was scored (P8b). +Second dev run (P8a-2, 13 Nov 2026, after C-210 and C-211): experiment p8a-dev-harness-check-2 + version 1, protocol_version draft, id ec2b4fa8-4398-5f70-be4c-5edf9cd1e0ef (the first run is kept + as it is); DAIL e1bde5e (clean tree); TerraPreserve and manifest as above; scope policies + dail-scope-2, stateful-full-scope-2, stateless-scope-2. 896 trials, 896 COMPLETED, 0 FAILED, + 0 harness or input failures; wall time 330 s. + Transitions evaluated / TRUNCATED: A 280 / 0, B 273 / 7, C 273 / 7, C' 273 / 7. + Transitions with an open target: A 56, B 56, C 56, C' 56. + Work (gate, invariant, applicable baseline, not-applicable baseline; of the invariant items, + NOT_PROTECTED_IN_PARENT): A 560, 140, 0, 0, 0; B 546, 1092, 56, 10, 0; C 546, 681, 56, 10, 54; + C' 546, 681, 62, 16, 54 (U_t items in 53 transitions each under C and C'). + Decisions per condition and family (PROMOTE/REJECT; no RETRY or ESCALATE): A and B as in the + first run; C and C' as in the first run except F8: C 63/14, C' 63/14. + Changed against the first run (same scenario, condition and step; counts only): A 0, B 0, + C 77, C' 77, all in F8: REJECT -> PROMOTE 28, TRUNCATED -> PROMOTE 35, TRUNCATED -> REJECT 14, + for each of C and C'. Nothing was tuned to the outcome. +Mechanism found in the first run (C-209, closed by C-210): under C and C' every F8 trajectory is rejected at step 1 with + INSUFFICIENT_EVIDENCE on the open target, because impact does not list it and a VIOLATED + reference cannot carry; C-196's stated purpose is not reached there. Nothing was tuned. +Tests: tests.unit.test_p8a_open_targets, test_p8a_open_targets_e2e, test_p8a_conditions, + test_experiment_loader, test_experiment_harness, test_experiment_terrapreserve (opt-in, + TERRAPRESERVE_ROOT), tests.contract.test_experiment_blinding; test_identity_strategy amended + (C-208). Mutation checks: 18 of 18 harness, loader and promotion mutants caught (M5 only after a + test was added), and 2 of 2 for the amended identity test. +Limits: the open-target rule blocked C and C' on F8 (C-209, closed by C-210 in P8a-2); condition-added work items + carry a stand-in reason code (C-207 (1)); automatic promotion is enabled by the reviewed + experiments/config/promotion.toml in the harness's copy of the configuration only (C-207 (3)); + repetition 1, no seeds; one machine. +Carry-forward: C-200 to C-204 and C-207 Proposed; C-209 closed by C-210; P8b scores the stored records. +``` + +## P8b–P10 — NOT STARTED P9 (AWS integration) and milestone M6 (AWS Demonstration) are removed (C-71, Decided by the advisor, 11 Nov 2026): AWS stays fully offline, with no deployment, no AWS account and no calibration against diff --git a/experiments/README.md b/experiments/README.md new file mode 100644 index 0000000..de6923c --- /dev/null +++ b/experiments/README.md @@ -0,0 +1,80 @@ +# experiments — the P8 experiment harness + +Runs TerraPreserve target scenarios through DAIL under the four conditions of C-198 and stores what +happened. It never bypasses DAIL's verification and promotion path: each transition is one +`EvaluationOrchestrator.evaluate` call. P8a builds the harness; scoring, metrics and reports are +P8b. + +## Running + +The run is read-only towards TerraPreserve. It is free and offline: there is no network, no LLM, +no AWS, and Terraform is never run. + +```bash +export TERRAPRESERVE_ROOT=/path/to/TerraPreserve # a checkout at tag terrapreserve-v1.0 +export DAIL_EXPERIMENT_ROOT=/path/outside/both/repos # where output goes; never committed +python3.12 -m experiments run --name p8a-dev-harness-check --protocol-version draft +python3.12 -m experiments summary --name p8a-dev-harness-check +``` + +- **Options:** `--conditions A,B,C,C_PRIME` (the default is all four), `--only ` and + `--workers N` (trials run in threads; each has its own store). +- **Interruptions:** running the same command again resumes. COMPLETED and FAILED trials are + skipped. An interrupted trial is marked ABORTED and re-run as a new attempt. Nothing stored is + overwritten. +- **Refusals:** `run` refuses to start in any of these cases: + - TerraPreserve is not at `terrapreserve-v1.0`; + - the manifest's SHA-256 is not the pinned + `665b326a024d295f727cd5648ffc7e0d375b9300584871f238c6bfa48edd2e2d`; + - any scenario or step hash does not recompute; + - `DAIL_EXPERIMENT_ROOT` is inside either repository. + +## The conditions (C-198) + +| Condition | Identity | Scope policy | Promotion policy | What the lineage tracks | +|---|---|---|---|---| +| A, Stateless | evidence | `StatelessScopePolicy(target)` | `stateless_policy(target)` | the repair target only; nothing for a change request (C-205, C-206) | +| B, Stateful Full | evidence | `StatefulFullScopePolicy` | `stateful_full_policy` | the four invariants | +| C, DAIL | evidence | `DailScopePolicy` | `dail_policy` | the four invariants | +| C′, DAIL without identity evidence | `AddressOnlyIdentity` | `DailScopePolicy` | `dail_policy` | the four invariants | + +- **Registry:** every condition uses invariant registry v1.1, stored as + `invariant_registry_version` 2 (C-180, C-203). +- **Open targets:** on a scenario with more than one step after S0 (F8), the repair target is an + open target on every step but the last (C-196). +- **Early termination:** a trajectory stops at the first decision other than PROMOTE, and the + remaining steps are recorded as TRUNCATED (C-200). +- **Promotion switch:** automatic promotion is enabled only in the harness's own copy of the + configuration, by the reviewed file `experiments/config/promotion.toml` (C-207). `config/` keeps + it off in every environment. + +## Where output goes (Doc 13 §41) + +```text +$DAIL_EXPERIMENT_ROOT/experiment// + definition/experiment.json the definition and its pins (Doc 13 §5, §46) + fixtures/fixtures.json the scenario and step hashes (§40) + trials//events.jsonl the lifecycle (§7), append-only + trials//attempts// + store/dail.sqlite the attempt's own DAIL store (C-202) + baseline/ generation/ analysis/ verification/ decision/ metrics/ + result.json the trial result, with no wall-clock time + timing.json the §6 Trial record with its wall-clock times +``` + +## Blinding and the test split + +- **Labels:** trial execution never reads TerraPreserve labels. The loader gives execution only + these fields: + - the scenario id, family, split and template id; + - the step directories and their hashes; + - the target's kind and invariant id; + - the protected invariants; + - the endpoints; + - the primary variable values. + + `tests/contract/test_experiment_blinding.py` fails if any module here names `labels`, + `gt_class`, `oracle`, `mutation_record`, `intent` or `variable_values_alt`. Labels are used only + for scoring, which is P8b. +- **Test split:** the loader refuses `split = "test"`. The test split, `$TERRAPRESERVE_PRIVATE` and + the test ids are not read before P10, after the protocol is frozen. diff --git a/experiments/__init__.py b/experiments/__init__.py new file mode 100644 index 0000000..257edf1 --- /dev/null +++ b/experiments/__init__.py @@ -0,0 +1,19 @@ +"""The experiment harness (P8a; Doc 13; C-196 to C-207). + +It runs TerraPreserve target scenarios through DAIL under the four conditions of C-198 (A Stateless, +B Stateful Full, C DAIL, C' DAIL without identity evidence) and stores what happened. It never +bypasses the deterministic verification and promotion path (Doc 13 "Safety principle"): every +transition is one ``EvaluationOrchestrator.evaluate`` call, and only the Promotion Controller moves +a trial's Trusted State. + +Modules: ``loader`` (TerraPreserve read-only, hashes verified, execution fields only), +``definition`` (Experiment, Trial, conditions, lifecycle), ``conditions`` (what each condition +passes to DAIL), ``trial`` (one attempt of one trial), ``records`` (what is stored per transition), +``artifacts`` (write-once files), ``runner`` (plan, run, resume), ``__main__`` (the command line). + +Blinding: trial execution never reads oracle labels; ``tests/contract/test_experiment_blinding.py`` +parses every module here. Scoring against labels, metrics and reports are P8b. The test split is +never read before P10 (the loader refuses it). + +Not part of ``core``: nothing in ``core`` or ``evidence`` imports this package (rule R3). +""" diff --git a/experiments/__main__.py b/experiments/__main__.py new file mode 100644 index 0000000..cb9bb6a --- /dev/null +++ b/experiments/__main__.py @@ -0,0 +1,185 @@ +"""The experiment harness command line (P8a; C-204). + + python -m experiments run --name NAME [--version V] [--protocol-version draft] + [--conditions A,B,C,C_PRIME] [--only ID,...] [--workers N] + python -m experiments summary --name NAME [--version V] + +``TERRAPRESERVE_ROOT`` names a TerraPreserve checkout, read only; it must be at tag +``terrapreserve-v1.0`` (checked with ``git describe``) and its manifest must have the pinned hash. +``DAIL_EXPERIMENT_ROOT`` is where output goes; it must be outside this repository and outside the +TerraPreserve checkout, and it is never committed. The test split is never read. + +``summary`` prints counts read from the stored records, never a label: trials by status, decisions +per condition and family, transitions evaluated and TRUNCATED per condition, transitions with an +open target, work counts per condition, harness and input failures, and wall time. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from collections import Counter, defaultdict +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from experiments.artifacts import read_record +from experiments.definition import Condition, ExperimentSpec, experiment_id_for +from experiments.loader import TERRAPRESERVE_TAG, load +from experiments.runner import REPO_ROOT, Layout, plan, read_events, read_result, run_experiment + + +class CommandError(Exception): + pass + + +def _env_path(name: str) -> Path: + value = os.environ.get(name) + if not value: + raise CommandError(f"set {name}") + return Path(value).resolve() + + +def _inside(path: Path, root: Path) -> bool: + return path == root or root in path.parents + + +def _git(root: Path, *args: str) -> str: + done = subprocess.run( + ["git", "-C", str(root), *args], capture_output=True, text=True, check=False + ) + if done.returncode != 0: + raise CommandError(f"git {' '.join(args)} failed in {root}: {done.stderr.strip()}") + return done.stdout.strip() + + +def _roots() -> tuple[Path, Path]: + terrapreserve = _env_path("TERRAPRESERVE_ROOT") + output = _env_path("DAIL_EXPERIMENT_ROOT") + for root in (REPO_ROOT.resolve(), terrapreserve): + if _inside(output, root): + raise CommandError(f"DAIL_EXPERIMENT_ROOT must be outside {root}") + return terrapreserve, output + + +def _environment(terrapreserve: Path) -> dict[str, str]: + tag = _git(terrapreserve, "describe", "--tags", "--exact-match") + if tag != TERRAPRESERVE_TAG: + raise CommandError(f"TerraPreserve is at {tag!r}, not {TERRAPRESERVE_TAG}") + return { + "dail_commit": _git(REPO_ROOT, "rev-parse", "HEAD"), + "dail_worktree_clean": str(_git(REPO_ROOT, "status", "--porcelain") == "").lower(), + "terrapreserve_tag": tag, + "terrapreserve_commit": _git(terrapreserve, "rev-parse", "HEAD"), + } + + +def _spec(args: argparse.Namespace) -> ExperimentSpec: + return ExperimentSpec( + name=args.name, + version=args.version, + protocol_version=args.protocol_version, + objective="P8a harness check on the TerraPreserve v1.0 dev targets (no result is claimed)", + hypotheses=("none: a harness check, not an experiment result",), + conditions=tuple(Condition(c) for c in args.conditions.split(",")), + ) + + +def cmd_run(args: argparse.Namespace) -> int: + terrapreserve, output = _roots() + environment = _environment(terrapreserve) + loaded = load(terrapreserve) + only = None if not args.only else args.only.split(",") + experiment = plan( + _spec(args), loaded, environment=environment, created_at=datetime.now(UTC), only=only + ) + summary = run_experiment(experiment, loaded, output, workers=args.workers) + counts = Counter(s.value for s in summary.statuses.values()) + print(json.dumps({"experiment_id": experiment.experiment_id, "statuses": counts}, indent=1)) + for trial_id, message in summary.failures.items(): + print(f"FAILED {trial_id}: {message}") + return 0 if not summary.failures else 1 + + +def summarize(layout: Layout) -> dict[str, Any]: + """Counts from the stored records of an experiment (no label is read).""" + definition = read_record(layout.definition_path) + trials = definition["trials"] + statuses: Counter[str] = Counter() + decisions: dict[str, Counter[str]] = defaultdict(Counter) + transitions: dict[str, Counter[str]] = defaultdict(Counter) + work: dict[str, Counter[str]] = defaultdict(Counter) + opened: Counter[str] = Counter() + failures: list[dict[str, Any]] = [] + wall = 0.0 + for trial in trials: + tid, condition = trial["trial_id"], trial["condition"] + events = read_events(layout, tid) + statuses[events[-1]["status"] if events else "NOT_STARTED"] += 1 + try: + result = read_result(layout, tid) + except Exception: # not run yet + continue + attempt = layout.attempt_dir(tid, result["attempt"]) + timing = read_record(attempt / "timing.json") + wall += float(timing["durations_seconds"]["total"]) + if result["failure"] is not None: + failures.append( + {"trial_id": tid, "scenario_id": result["scenario_id"], **result["failure"]} + ) + if result["baseline"] is not None and result["baseline"]["outcome"] == "REFUSED": + decisions[f"{condition}|{result['family']}"]["BASELINE_REFUSED"] += 1 + for t in result["transitions"]: + transitions[condition][t["status"]] += 1 + if t["status"] != "EVALUATED": + continue + decisions[f"{condition}|{result['family']}"][t["decision"]["type"]] += 1 + if t["open_targets"]: + opened[condition] += 1 + for key, value in t["work_counts"].items(): + work[condition][key] += value + return { + "experiment_id": layout.experiment_id, + "trials": len(trials), + "trials_by_status": dict(statuses), + "decisions_by_condition_and_family": {k: dict(v) for k, v in sorted(decisions.items())}, + "transitions_by_condition": {k: dict(v) for k, v in sorted(transitions.items())}, + "open_target_transitions_by_condition": dict(sorted(opened.items())), + "work_counts_by_condition": {k: dict(v) for k, v in sorted(work.items())}, + "failures": failures, + "trial_wall_time_seconds_sum": round(wall, 1), + } + + +def cmd_summary(args: argparse.Namespace) -> int: + output = _env_path("DAIL_EXPERIMENT_ROOT") + layout = Layout(output, experiment_id_for(args.name, args.version)) + print(json.dumps(summarize(layout), indent=1, sort_keys=True)) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m experiments") + commands = parser.add_subparsers(dest="command", required=True) + for name in ("run", "summary"): + command = commands.add_parser(name) + command.add_argument("--name", required=True) + command.add_argument("--version", default="1") + if name == "run": + command.add_argument("--protocol-version", default="draft") + command.add_argument("--conditions", default="A,B,C,C_PRIME") + command.add_argument("--only", default="") + command.add_argument("--workers", type=int, default=1) + args = parser.parse_args(argv) + try: + return cmd_run(args) if args.command == "run" else cmd_summary(args) + except CommandError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/artifacts.py b/experiments/artifacts.py new file mode 100644 index 0000000..85acf38 --- /dev/null +++ b/experiments/artifacts.py @@ -0,0 +1,124 @@ +"""Write-once experiment artifacts (P8a; Doc 13 §40, §41, §47; C-202, C-204). + +Layout (§41), under ``$DAIL_EXPERIMENT_ROOT``:: + + experiment// + definition/experiment.json the definition and its pins (§5, §46) + fixtures/fixtures.json scenario and step hashes (§40) + trials// + events.jsonl the trial's lifecycle (§7), append-only, wall clock + attempts// one attempt (§47: a re-run is a new attempt) + store/dail.sqlite the attempt's own DAIL store (C-202) + baseline/ generation/ analysis/ verification/ decision/ metrics/ + result.json the trial result (no wall clock) + timing.json the Trial record of §6 with its wall-clock times + report/ P8b + +§41 names ``trials//{baseline/ ...}``; the attempts level is added so that a re-run never +overwrites the original (§47) (C-207). Every file is written once: ``write_record`` refuses a path +that exists, and ``append_event`` only appends. JSON is written with sorted keys, so the same record +gives the same bytes. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Final + +ATTEMPT_DIRS: Final = ("baseline", "generation", "analysis", "verification", "decision", "metrics") + + +class ArtifactError(Exception): + """A record would be overwritten, or a stored record cannot be read.""" + + +def dumps(data: Any) -> bytes: + return (json.dumps(data, sort_keys=True, indent=1, ensure_ascii=False) + "\n").encode("utf-8") + + +def write_record(path: Path, data: Any) -> Path: + """Write ``data`` to ``path`` once; an existing path is never overwritten.""" + path.parent.mkdir(parents=True, exist_ok=True) + try: + with path.open("xb") as handle: + handle.write(dumps(data)) + handle.flush() + os.fsync(handle.fileno()) + except FileExistsError: + raise ArtifactError(f"{path} exists; a stored record is never overwritten") from None + return path + + +def write_or_same(path: Path, data: Any) -> bool: + """Write ``data`` once, or check that the stored record is the same. True if written.""" + if path.exists(): + if path.read_bytes() != dumps(data): + raise ArtifactError(f"{path} holds another record") + return False + write_record(path, data) + return True + + +def read_record(path: Path) -> Any: + try: + return json.loads(path.read_bytes()) + except (OSError, ValueError) as error: + raise ArtifactError(f"{path}: {error}") from None + + +def append_event(path: Path, event: dict[str, Any]) -> None: + """Append one line to an append-only JSON-lines file.""" + path.parent.mkdir(parents=True, exist_ok=True) + line = json.dumps(event, sort_keys=True, ensure_ascii=False) + "\n" + with path.open("ab") as handle: + handle.write(line.encode("utf-8")) + handle.flush() + os.fsync(handle.fileno()) + + +def read_events(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + events = [] + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + events.append(json.loads(line)) + except ValueError as error: + raise ArtifactError(f"{path} line {number}: {error}") from None + return events + + +class Layout: + """The paths of one experiment under an output root.""" + + def __init__(self, root: Path, experiment_id: str) -> None: + self.root = Path(root) + self.experiment_id = experiment_id + self.experiment_dir = self.root / "experiment" / experiment_id + + @property + def definition_path(self) -> Path: + return self.experiment_dir / "definition" / "experiment.json" + + @property + def fixtures_path(self) -> Path: + return self.experiment_dir / "fixtures" / "fixtures.json" + + def trial_dir(self, trial_id: str) -> Path: + return self.experiment_dir / "trials" / trial_id + + def events_path(self, trial_id: str) -> Path: + return self.trial_dir(trial_id) / "events.jsonl" + + def attempt_dir(self, trial_id: str, attempt: int) -> Path: + return self.trial_dir(trial_id) / "attempts" / str(attempt) + + def attempts(self, trial_id: str) -> list[int]: + base = self.trial_dir(trial_id) / "attempts" + if not base.is_dir(): + return [] + return sorted(int(p.name) for p in base.iterdir() if p.is_dir() and p.name.isdigit()) diff --git a/experiments/conditions.py b/experiments/conditions.py new file mode 100644 index 0000000..c8fe568 --- /dev/null +++ b/experiments/conditions.py @@ -0,0 +1,199 @@ +"""What each condition gives DAIL (P8a; C-196 to C-198, C-203, C-205, C-206). + +All four conditions use the same verifiers, the same ``EvaluationOrchestrator`` and the same +Promotion Controller (RV-001); they differ only in the scope policy, the policy constructor, the +identity strategy and, for A, the invariants its lineage tracks: + +===== ================ ========================= ======================= ====================== +cond. identity scope policy policy constructor lineage tracks +===== ================ ========================= ======================= ====================== +A EvidenceIdentity StatelessScopePolicy(t) stateless_policy(t) t only (C-205, C-206) +B EvidenceIdentity StatefulFullScopePolicy stateful_full_policy the four invariants +C EvidenceIdentity DailScopePolicy dail_policy the four invariants +C' AddressOnly DailScopePolicy dail_policy the four invariants +===== ================ ========================= ======================= ====================== + +``t`` is the repair target of a ``repair`` scenario, or nothing for a ``change_request`` (C-197). + +Registry (C-203): every condition uses registry v1.1 (``registry_v1_1_invariants``, INV-SEC-001/002 +version 2), stored as ``invariant_registry_version`` 2 (C-180). TerraPreserve names the protected +invariants by their registry-v1.0 ids at version "1.0"; each maps to the v1.1 definition of the same +id. A store registers an invariant's earlier versions first (``history``): Doc 06 §4.2 allows no +gaps in an invariant's versions, so version 2 of INV-SEC-001 needs version 1 registered before it. + +Open targets (C-196): on a scenario with more than one step after S0, the repair target is open on +every step but the last. +""" + +from __future__ import annotations + +import functools +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Final + +from core.application.scope_policy import ( + DailScopePolicy, + StatefulFullScopePolicy, + StatelessScopePolicy, + VerificationScopePolicy, +) +from core.dependency.invariants import SemanticBinding +from core.domain.invariant import Invariant +from core.domain.promotion import PromotionPolicy +from core.domain.registry_v1 import registry_v1_invariants +from core.domain.registry_v1_1 import ( + REGISTRY_TAG_V1_1, + REGISTRY_VERSION_V1_1, + registry_v1_1_invariants, +) +from core.identity.strategy import AddressOnlyIdentity, EvidenceIdentity, IdentityStrategy +from core.promotion.policy import ( + STATELESS_RULES_VERSION, + dail_policy, + stateful_full_policy, + stateless_policy, +) +from experiments.definition import Condition +from experiments.loader import ScenarioInput + +REPAIR: Final = "repair" +CHANGE_REQUEST: Final = "change_request" +TERRAPRESERVE_VERSION: Final = "1.0" # the registry version TerraPreserve v1.0 names +FUNC_BINDING: Final = ("INV-FUNC-001", "TCP", 5432) +REGISTRY_RECORD: Final = { + "stored_invariant_registry_version": REGISTRY_VERSION_V1_1, + "means": REGISTRY_TAG_V1_1, +} +POLICY_RULES: Final = { + Condition.A: STATELESS_RULES_VERSION, + Condition.B: "stateful-full", + Condition.C: "dail", + Condition.C_PRIME: "dail", +} + + +class ConditionError(Exception): + """A scenario that a condition cannot be built for (a harness failure).""" + + +@dataclass(frozen=True) +class ConditionSetup: + """Everything a trial of ``condition`` on one scenario passes to DAIL.""" + + condition: Condition + policy_version: str + scope_policy: VerificationScopePolicy + policy_factory: Callable[..., PromotionPolicy] + identity_strategy: IdentityStrategy + definitions: tuple[Invariant, ...] # evaluation: graph, impact, verification + bindings: tuple[SemanticBinding, ...] + baseline_definitions: tuple[Invariant, ...] # what the lineage tracks + baseline_bindings: tuple[SemanticBinding, ...] + allow_no_invariants: bool + history: tuple[Invariant, ...] # earlier versions registered first (Doc 06 §4.2) + required: tuple[str, ...] | None # A's explicit required set, None for B, C, C' + protected_mapping: dict[str, dict[str, Any]] + + def open_targets(self, scenario: ScenarioInput, step: int) -> tuple[str, ...]: + """C-196: the repair target is open on steps 1..n-1 of a scenario with n > 1 steps.""" + if ( + scenario.target_kind == REPAIR + and scenario.target_invariant_id is not None + and scenario.transitions > 1 + and step < scenario.transitions + ): + return (scenario.target_invariant_id,) + return () + + def versions(self) -> dict[str, Any]: + return { + "policy_version": self.policy_version, + "policy_rules": POLICY_RULES[self.condition], + "scope_policy": getattr(self.scope_policy, "version", type(self.scope_policy).__name__), + "identity": self.identity_strategy.algorithm_version, + } + + +def _target(scenario: ScenarioInput) -> tuple[str, ...]: + if scenario.target_kind == REPAIR: + if scenario.target_invariant_id is None: + raise ConditionError(f"{scenario.scenario_id}: a repair scenario names no target") + return (scenario.target_invariant_id,) + if scenario.target_kind == CHANGE_REQUEST: + return () + raise ConditionError(f"{scenario.scenario_id}: unknown target kind {scenario.target_kind!r}") + + +def _mapping( + scenario: ScenarioInput, definitions: tuple[Invariant, ...] +) -> dict[str, dict[str, Any]]: + """C-203: TerraPreserve's protected invariants (v1.0 ids) to the v1.1 definitions.""" + by_id = {d.invariant_id: d for d in definitions} + found: dict[str, dict[str, Any]] = {} + for invariant_id, version in scenario.protected_invariants: + definition = by_id.get(invariant_id) + if definition is None or version != TERRAPRESERVE_VERSION: + raise ConditionError( + f"{scenario.scenario_id}: protected invariant {invariant_id} {version} has no " + "registry-v1.1 definition" + ) + found[invariant_id] = { + "terrapreserve_version": version, + "dail_version": definition.version, + "registry": REGISTRY_TAG_V1_1, + } + return found + + +def setup_for(condition: Condition, scenario: ScenarioInput, now: datetime) -> ConditionSetup: + """The setup of ``condition`` for ``scenario`` (see the module docstring).""" + source, destination = scenario.source_endpoint, scenario.destination_endpoint + definitions = registry_v1_1_invariants(now, source=source, destination=destination) + func, protocol, port = FUNC_BINDING + bindings = (SemanticBinding(func, source, destination, protocol, port),) + mapping = _mapping(scenario, definitions) + targets = _target(scenario) + if condition is Condition.A: + tracked = tuple(d for d in definitions if d.invariant_id in targets) + scope: VerificationScopePolicy = StatelessScopePolicy(targets) + factory: Callable[..., PromotionPolicy] = functools.partial( + stateless_policy, required=targets, invariant_registry_version=REGISTRY_VERSION_V1_1 + ) + required: tuple[str, ...] | None = targets + else: + tracked = definitions + full = condition is Condition.B + scope = StatefulFullScopePolicy() if full else DailScopePolicy() + factory = functools.partial( + stateful_full_policy if full else dail_policy, + invariant_registry_version=REGISTRY_VERSION_V1_1, + ) + required = None + identity: IdentityStrategy = ( + AddressOnlyIdentity() if condition is Condition.C_PRIME else EvidenceIdentity() + ) + tracked_ids = {d.invariant_id for d in tracked} + earlier = registry_v1_invariants(now, source=source, destination=destination) + history = tuple( + d + for d in earlier + if d.invariant_id in tracked_ids + and d.version < next(t.version for t in tracked if t.invariant_id == d.invariant_id) + ) + return ConditionSetup( + condition=condition, + policy_version=f"p8a-{condition.value}-{POLICY_RULES[condition]}", + scope_policy=scope, + policy_factory=factory, + identity_strategy=identity, + definitions=definitions, + bindings=bindings, + baseline_definitions=tracked, + baseline_bindings=tuple(b for b in bindings if b.invariant_id in tracked_ids), + allow_no_invariants=not tracked, + history=history, + required=required, + protected_mapping=mapping, + ) diff --git a/experiments/config/promotion.toml b/experiments/config/promotion.toml new file mode 100644 index 0000000..67761f4 --- /dev/null +++ b/experiments/config/promotion.toml @@ -0,0 +1,11 @@ +# Reviewed experiment configuration (P8a, C-207). Flat keys for the promotion area, as in +# config/base/promotion.toml, which it replaces only in the experiment harness's own copy of the +# repository configuration (experiments.runner.experiment_config). +# +# automatic_enabled is safety-critical (Doc 14 §37-38) and may be enabled only in a reviewed +# configuration file. It is enabled here, for the experiment harness only: every trial runs on its +# own fresh store and lineage under the experiment root, outside the repository, and no experiment +# lineage is ever the authoritative lineage (C-202, Doc 13 §20). Without it every PROMOTE would be an +# ESCALATE and no condition could be measured. The repository configuration (config/) is unchanged: +# automatic promotion stays off in every environment there. +automatic_enabled = true diff --git a/experiments/definition.py b/experiments/definition.py new file mode 100644 index 0000000..a124734 --- /dev/null +++ b/experiments/definition.py @@ -0,0 +1,204 @@ +"""Experiment and trial records (P8a; Doc 13 §5-§7, §39, §46, §48; C-198, C-200). + +Doc 13 §5 "Experiment { experiment_id, name, version, objective, hypothesis[], protocol_version, +fixture_set[], baseline_set[], condition_set[], repetitions, seed_policy, metrics[], +stopping_policy, created_at }. An experiment definition becomes immutable once trials begin. +Changes create a new experiment version." §6 "Trial { trial_id, experiment_id, condition_id, +fixture_id, baseline_id, repetition, seed | null, parent_state_id, status, started_at, +completed_at, artifact_refs[] }." +§7 "PLANNED -> INITIALIZED -> EXECUTING -> (FAILED | ABORTED | COMPLETED) -> ANALYZED -> INCLUDED." + +Ids are deterministic (name-based UUIDs): the experiment id from its name and version, a trial id +from the experiment id, the scenario id, the condition and the repetition. Two readings of §7, for +the authors (C-207): a trial interrupted before it began executing may go from INITIALIZED to +ABORTED (stale-run handling, §50), and an ABORTED trial is re-run as a new attempt, which starts +again at INITIALIZED (§47: a re-run never overwrites the original). ``pins`` holds what §46 asks +to pin. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from datetime import datetime +from enum import StrEnum +from typing import Any, Final + +from experiments.loader import FixtureRecord + +NAMESPACE: Final = uuid.uuid5(uuid.NAMESPACE_URL, "dail/experiments/p8a") +SEED_POLICY: Final = "none: fixed patches are deterministic, repetition 1 (C-200)" +STOPPING_POLICY: Final = ( + "every planned trial runs once; a trajectory stops at the first decision other than PROMOTE " + "and the rest of its steps are TRUNCATED (C-200)" +) + + +class Condition(StrEnum): + """The four conditions of P8 (C-198).""" + + A = "A" + B = "B" + C = "C" + C_PRIME = "C_PRIME" + + +CONDITION_NAMES: Final = { + Condition.A: "Stateless", + Condition.B: "Stateful Full", + Condition.C: "DAIL", + Condition.C_PRIME: "DAIL without identity evidence", +} + + +class TrialStatus(StrEnum): + """Doc 13 §7.""" + + PLANNED = "PLANNED" + INITIALIZED = "INITIALIZED" + EXECUTING = "EXECUTING" + FAILED = "FAILED" + ABORTED = "ABORTED" + COMPLETED = "COMPLETED" + ANALYZED = "ANALYZED" + INCLUDED = "INCLUDED" + + +S = TrialStatus +_NEXT: Final[Mapping[TrialStatus, frozenset[TrialStatus]]] = { + S.PLANNED: frozenset({S.INITIALIZED}), + S.INITIALIZED: frozenset({S.EXECUTING, S.ABORTED}), + S.EXECUTING: frozenset({S.FAILED, S.ABORTED, S.COMPLETED}), + S.ABORTED: frozenset({S.INITIALIZED}), # a new attempt (§47) + S.FAILED: frozenset(), + S.COMPLETED: frozenset({S.ANALYZED}), + S.ANALYZED: frozenset({S.INCLUDED}), + S.INCLUDED: frozenset(), +} +TERMINAL: Final = frozenset({S.FAILED, S.COMPLETED, S.ANALYZED, S.INCLUDED}) + + +class TransitionError(Exception): + """A trial status change that Doc 13 §7 does not allow.""" + + +def check_transition(before: TrialStatus, after: TrialStatus) -> None: + if after not in _NEXT[before]: + raise TransitionError(f"a trial cannot go from {before.value} to {after.value}") + + +def name_id(*parts: object) -> str: + return str(uuid.uuid5(NAMESPACE, "/".join(str(part) for part in parts))) + + +def experiment_id_for(name: str, version: str) -> str: + return name_id("experiment", name, version) + + +def trial_id_for( + experiment_id: str, scenario_id: str, condition: Condition, repetition: int +) -> str: + return name_id("trial", experiment_id, scenario_id, condition.value, repetition) + + +@dataclass(frozen=True) +class ExperimentSpec: + """What the caller asks for; ``plan`` turns it into an ``Experiment``.""" + + name: str + version: str + protocol_version: str + objective: str + hypotheses: tuple[str, ...] + conditions: tuple[Condition, ...] + repetitions: int = 1 + + def __post_init__(self) -> None: + for key in ("name", "version", "protocol_version", "objective"): + value = getattr(self, key) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"ExperimentSpec.{key} must be text") + conditions = tuple(Condition(c) for c in self.conditions) + if not conditions or len(set(conditions)) != len(conditions): + raise ValueError("ExperimentSpec.conditions: at least one, each once") + if self.repetitions != 1: + raise ValueError("ExperimentSpec.repetitions: fixed patches run once (C-200)") + object.__setattr__(self, "conditions", tuple(sorted(conditions))) + object.__setattr__(self, "hypotheses", tuple(self.hypotheses)) + + +@dataclass(frozen=True) +class PlannedTrial: + trial_id: str + scenario_id: str + condition: Condition + repetition: int + + +@dataclass(frozen=True) +class Experiment: + """Doc 13 §5, with the pins of §46 and the fixture hashes of §40.""" + + experiment_id: str + name: str + version: str + objective: str + hypothesis: tuple[str, ...] + protocol_version: str + fixture_set: tuple[str, ...] + baseline_set: tuple[tuple[str, str], ...] + condition_set: tuple[Condition, ...] + repetitions: int + created_at: datetime + pins: Mapping[str, Any] + fixtures: tuple[FixtureRecord, ...] + seed_policy: str = SEED_POLICY + metrics: tuple[str, ...] = () + stopping_policy: str = STOPPING_POLICY + planned: tuple[PlannedTrial, ...] = field(default=()) + + def trial_id(self, scenario_id: str, condition: Condition, repetition: int = 1) -> str: + return trial_id_for(self.experiment_id, scenario_id, condition, repetition) + + def to_dict(self) -> dict[str, Any]: + return { + "experiment_id": self.experiment_id, + "name": self.name, + "version": self.version, + "objective": self.objective, + "hypothesis": list(self.hypothesis), + "protocol_version": self.protocol_version, + "fixture_set": list(self.fixture_set), + "baseline_set": [{"scenario_id": s, "step_0": h} for s, h in self.baseline_set], + "condition_set": [c.value for c in self.condition_set], + "repetitions": self.repetitions, + "seed_policy": self.seed_policy, + "metrics": list(self.metrics), + "stopping_policy": self.stopping_policy, + "created_at": self.created_at.isoformat(), + "pins": dict(self.pins), + "trials": [ + { + "trial_id": t.trial_id, + "scenario_id": t.scenario_id, + "condition": t.condition.value, + "repetition": t.repetition, + } + for t in self.planned + ], + } + + def fixtures_dict(self) -> dict[str, Any]: + return {"fixtures": [f.to_dict() for f in self.fixtures]} + + +def planned_trials( + experiment_id: str, scenario_ids: Iterable[str], conditions: Iterable[Condition] +) -> tuple[PlannedTrial, ...]: + chosen = tuple(conditions) + return tuple( + PlannedTrial(trial_id_for(experiment_id, sid, c, 1), sid, c, 1) + for sid in sorted(scenario_ids) + for c in chosen + ) diff --git a/experiments/loader.py b/experiments/loader.py new file mode 100644 index 0000000..62288c4 --- /dev/null +++ b/experiments/loader.py @@ -0,0 +1,273 @@ +"""The TerraPreserve loader (P8a; work package §3, §4.2; C-201, C-204; ADR-010, ADR-013). + +Read only. It never imports, copies or runs TerraPreserve code; it reads the frozen files of a +TerraPreserve checkout: ``scenarios/V1_MANIFEST.json`` and ``scenarios/dev// +{scenario.json, step_0/, step_1/, ...}`` (D4-09). + +**Checks, all before anything is returned** (one failure raises ``LoaderError`` and nothing partial +comes back, so a mismatch stops the run before any trial starts): + +1. the split is ``dev``; ``test`` is refused (the test split is touched once, in P10, after the + protocol is frozen), and so is anything else; +2. the SHA-256 of the manifest's bytes is the pin (``MANIFEST_SHA256`` for TerraPreserve v1.0); +3. for every manifest entry of the split with a wanted ``v1_role`` (``target`` by default): the + SHA-256 of ``scenario.json`` is the manifest's ``scenario_json_sha256``; the scenario names + itself and the ``dev`` split; its steps are 0..n in order, the same as the manifest's; every + step's ``content_hash``, recomputed here from the definition of D4-09, equals both the manifest's + and the scenario's. + +**What execution gets** (``ScenarioInput``): the scenario id, family, split, template id or source +lineage id (a real-world scenario has no template), the step directories and hashes, the target's +kind and invariant id, the protected invariants, the two endpoints and the primary variable values. +The loader reads exactly those keys and nothing else of a scenario; the manifest's list of the +other split is never indexed. + +``content_hash`` (D4-09): the SHA-256 of the newline-joined, path-sorted lines +``":"`` of the step directory (no +trailing newline). +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Any, Final + +MANIFEST_SHA256: Final = "665b326a024d295f727cd5648ffc7e0d375b9300584871f238c6bfa48edd2e2d" +TERRAPRESERVE_TAG: Final = "terrapreserve-v1.0" +DEV_SPLIT: Final = "dev" +REFUSED_SPLIT: Final = "test" +# C-201: the admission evidence of a step of the pinned v1.0 manifest (TerraPreserve D6 gate 2). +D6_VALIDATE_EVIDENCE: Final = ( + "terraform validate PASS at TerraPreserve D6 gate 2 (docs/D6_GATES.md: Terraform 1.15.8, " + "hashicorp/aws 6.53.0, every step of every target); not run by DAIL (C-201)" +) +NO_VALIDATE_EVIDENCE: Final = ( + "no terraform validate evidence: the manifest is not the pinned TerraPreserve v1.0 manifest" +) + + +class LoaderError(Exception): + """The TerraPreserve input cannot be used: nothing was loaded (an input failure).""" + + +@dataclass(frozen=True) +class StepInput: + """One step of a scenario: its directory and its verified content hash.""" + + index: int + directory: Path + content_hash: str + + +@dataclass(frozen=True) +class ScenarioInput: + """Everything trial execution may know about a scenario (work package §4.2).""" + + scenario_id: str + family: str + split: str + template_id: str | None + source_lineage_id: str | None + target_kind: str + target_invariant_id: str | None + protected_invariants: tuple[tuple[str, str], ...] + source_endpoint: str + destination_endpoint: str + variable_values: Mapping[str, Any] + steps: tuple[StepInput, ...] + + @property + def transitions(self) -> int: + """The number of steps after S0.""" + return len(self.steps) - 1 + + +@dataclass(frozen=True) +class FixtureRecord: + """The hashes of one scenario, for the experiment's fixture record (Doc 13 §40).""" + + scenario_id: str + scenario_json_sha256: str + step_hashes: tuple[str, ...] + v1_role: str + + def to_dict(self) -> dict[str, Any]: + return { + "scenario_id": self.scenario_id, + "scenario_json_sha256": self.scenario_json_sha256, + "step_hashes": list(self.step_hashes), + "v1_role": self.v1_role, + } + + +@dataclass(frozen=True) +class LoadedSet: + """A verified load: the scenarios for execution and the fixture hashes for the record.""" + + root: Path + split: str + manifest_sha256: str + scenarios: tuple[ScenarioInput, ...] + fixtures: tuple[FixtureRecord, ...] + admission_evidence: str + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def step_content_hash(directory: Path) -> str: + """D4-09 (see the module docstring). Refuses a symbolic link.""" + entries: list[tuple[str, Path]] = [] + for path in directory.rglob("*"): + if path.is_symlink(): + raise LoaderError(f"{directory}: symbolic link {path.name} is refused") + if path.is_file(): + entries.append((path.relative_to(directory).as_posix(), path)) + entries.sort(key=lambda entry: entry[0]) + lines = [f"{relative}:{sha256_hex(path.read_bytes())}" for relative, path in entries] + return sha256_hex("\n".join(lines).encode("utf-8")) + + +def _take(data: object, key: str, kind: type | tuple[type, ...], where: str) -> Any: + if not isinstance(data, Mapping) or key not in data: + raise LoaderError(f"{where}: no {key!r}") + value = data[key] + if not isinstance(value, kind): + raise LoaderError(f"{where}: {key!r} has the wrong type") + return value + + +def _steps(sid: str, base: Path, listed: object, manifest_steps: object) -> tuple[StepInput, ...]: + if not isinstance(listed, list) or not isinstance(manifest_steps, list) or not listed: + raise LoaderError(f"{sid}: steps must be a non-empty list in the scenario and the manifest") + if len(listed) != len(manifest_steps): + raise LoaderError(f"{sid}: the scenario and the manifest list different steps") + found = [] + for k, (step, pinned) in enumerate(zip(listed, manifest_steps, strict=True)): + where = f"{sid} step {k}" + if _take(step, "step", int, where) != k or _take(pinned, "step", int, where) != k: + raise LoaderError(f"{where}: steps must be 0..n in order") + name = _take(step, "terraform_dir", str, where) + directory = base / name + if Path(name).name != name or not directory.is_dir(): + raise LoaderError(f"{where}: step directory {name!r} is missing or not inside it") + recomputed = step_content_hash(directory) + claimed = _take(step, "content_hash", str, where) + manifest_hash = _take(pinned, "content_hash", str, where) + if not recomputed == claimed == manifest_hash: + raise LoaderError( + f"{where}: content_hash mismatch (recomputed {recomputed}, scenario {claimed}, " + f"manifest {manifest_hash})" + ) + found.append(StepInput(k, directory, recomputed)) + return tuple(found) + + +def _scenario(root: Path, split: str, entry: object) -> tuple[ScenarioInput, FixtureRecord]: + sid = _take(entry, "scenario_id", str, "manifest entry") + role = _take(entry, "v1_role", str, sid) + base = root / "scenarios" / split / sid + path = base / "scenario.json" + if Path(sid).name != sid or not path.is_file(): + raise LoaderError(f"{sid}: scenario.json is missing") + raw = path.read_bytes() + pinned = _take(entry, "scenario_json_sha256", str, sid) + if sha256_hex(raw) != pinned: + raise LoaderError(f"{sid}: scenario.json sha256 is not the manifest's") + try: + doc = json.loads(raw) + except ValueError as error: + raise LoaderError(f"{sid}: scenario.json is not JSON: {error}") from None + if _take(doc, "scenario_id", str, sid) != sid: + raise LoaderError(f"{sid}: scenario.json names another scenario") + if _take(doc, "split", str, sid) != split: + raise LoaderError(f"{sid}: scenario.json is not in the {split} split") + target = _take(doc, "target", dict, sid) + endpoints = _take(doc, "endpoints", dict, sid) + protected = [] + for item in _take(doc, "protected_invariants", list, sid): + protected.append((_take(item, "invariant_id", str, sid), _take(item, "version", str, sid))) + target_invariant = _take(target, "invariant_id", (str, type(None)), f"{sid} target") + steps = _steps(sid, base, _take(doc, "steps", list, sid), _take(entry, "steps", list, sid)) + values = _take(doc, "variable_values", dict, sid) + # a generated scenario names its template, a real-world one its source repository's lineage + template = doc.get("template_id") + source = doc.get("source") + lineage = source.get("repo_lineage_id") if isinstance(source, Mapping) else None + if not isinstance(template, str | None) or not isinstance(lineage, str | None): + raise LoaderError(f"{sid}: template_id and source.repo_lineage_id must be text") + if template is None and lineage is None: + raise LoaderError(f"{sid}: neither a template_id nor a source.repo_lineage_id") + scenario = ScenarioInput( + scenario_id=sid, + family=_take(doc, "family", str, sid), + split=split, + template_id=template, + source_lineage_id=lineage, + target_kind=_take(target, "kind", str, f"{sid} target"), + target_invariant_id=target_invariant, + protected_invariants=tuple(sorted(protected)), + source_endpoint=_take(endpoints, "S", str, f"{sid} endpoints"), + destination_endpoint=_take(endpoints, "D", str, f"{sid} endpoints"), + variable_values=MappingProxyType(dict(values)), + steps=steps, + ) + fixture = FixtureRecord(sid, pinned, tuple(s.content_hash for s in steps), role) + return scenario, fixture + + +def load( + root: Path, + *, + split: str = DEV_SPLIT, + manifest_sha256: str = MANIFEST_SHA256, + roles: Iterable[str] = ("target",), +) -> LoadedSet: + """Load and verify the ``split`` scenarios of the TerraPreserve checkout at ``root`` (see the + module docstring). Raises ``LoaderError`` on any refusal or mismatch.""" + if split == REFUSED_SPLIT: + raise LoaderError( + "the test split is refused: it is read once, in P10, after the protocol is frozen" + ) + if split != DEV_SPLIT: + raise LoaderError(f"split {split!r} is refused: only {DEV_SPLIT!r} is read in P8") + wanted = frozenset(roles) + manifest = Path(root) / "scenarios" / "V1_MANIFEST.json" + if not manifest.is_file(): + raise LoaderError(f"no TerraPreserve manifest at {manifest}") + raw = manifest.read_bytes() + digest = sha256_hex(raw) + if digest != manifest_sha256: + raise LoaderError(f"the manifest sha256 {digest} is not the pinned {manifest_sha256}") + try: + data = json.loads(raw) + except ValueError as error: + raise LoaderError(f"the manifest is not JSON: {error}") from None + entries = _take(data, split, list, "manifest") + scenarios, fixtures = [], [] + for entry in entries: + if _take(entry, "v1_role", str, "manifest entry") not in wanted: + continue + scenario, fixture = _scenario(Path(root), split, entry) + scenarios.append(scenario) + fixtures.append(fixture) + ids = [s.scenario_id for s in scenarios] + if len(set(ids)) != len(ids): + raise LoaderError("the manifest lists a scenario twice") + order = sorted(range(len(ids)), key=lambda n: ids[n]) + return LoadedSet( + root=Path(root), + split=split, + manifest_sha256=digest, + scenarios=tuple(scenarios[n] for n in order), + fixtures=tuple(fixtures[n] for n in order), + admission_evidence=( + D6_VALIDATE_EVIDENCE if digest == MANIFEST_SHA256 else NO_VALIDATE_EVIDENCE + ), + ) diff --git a/experiments/records.py b/experiments/records.py new file mode 100644 index 0000000..9ce5940 --- /dev/null +++ b/experiments/records.py @@ -0,0 +1,260 @@ +"""What a trial stores per transition (P8a; work package §4.3; Doc 13 §40, §43-§44). + +A transition record holds: the step index; the parent state id and hash; the candidate hash; the +open targets; the decision type and reason; the required set, and how each required item was proven +(VERIFIED, CARRIED_FORWARD) or which blocking conditions name it; every work item run, with its +class, target, result, ``applicable`` flag and origin (``impact``: impact listed it; ``condition``: +the condition's scope policy added it; ``gate``: the engine's mandatory gates); the work counts +split into gate items, invariant items, applicable baseline items and not-applicable baseline items; +the identity report's pairs with change types and the ``moved`` flag; the impact report's affected +invariants with impact types; the new state. Durations are kept apart, with the wall clock +(``timing.json``), so that a record holds no time that differs between two runs. + +Outcome classes (Doc 13 §43-§44): ``DAIL_OUTCOME`` (a decision; a construction failure counts as a +REJECT, and DAIL stores no decision for it, C-170 (3)), ``HARNESS_FAILURE`` (an exception in the +harness: the trial is FAILED and no decision is made up), ``INPUT_FAILURE`` (a hash mismatch). + +Every record carrying an invariant version carries ``registry`` (stored version 2 = +registry-v1.1, C-180, C-203). +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any, Final + +from core.application.orchestrator import EvaluationResult, ResultKind +from core.domain.impact import ImpactReport, ReasonCode, VerificationWorkItem +from core.domain.state import TrustedState +from core.domain.verification import VerificationReport +from experiments.conditions import REGISTRY_RECORD + +DAIL_OUTCOME: Final = "DAIL_OUTCOME" +HARNESS_FAILURE: Final = "HARNESS_FAILURE" +INPUT_FAILURE: Final = "INPUT_FAILURE" +NOT_EVALUATED: Final = "NOT_EVALUATED" +EVALUATED: Final = "EVALUATED" +TRUNCATED: Final = "TRUNCATED" +GATE_TYPE: Final = "STRUCTURAL" +NOT_PROTECTED_IN_PARENT: Final = ReasonCode.NOT_PROTECTED_IN_PARENT.value +# A random id is drawn for these reports (they are not derived from the request); the stored +# copy leaves it out and keeps the content hash, so two runs store the same bytes (C-207). +_RANDOM_IDS: Final = ("report_id", "graph_id") + + +def stable(data: Mapping[str, Any]) -> dict[str, Any]: + return {k: v for k, v in data.items() if k not in _RANDOM_IDS} + + +def state_summary(state: TrustedState) -> dict[str, Any]: + return { + "state_id": state.state_id, + "state_hash": state.state_hash, # covers the lineage (C-33): differs between trials + "resource_set_hash": state.resource_set_hash(), + "version": state.version, + "invariant_registry_version": state.invariant_registry_version, + "invariants": { + ref.invariant_id: {"version": ref.invariant_version, "status": ref.status.value} + for ref in sorted(state.invariant_refs, key=lambda r: r.invariant_id) + }, + } + + +def _counts() -> dict[str, int]: + return { + "gate": 0, + "invariant": 0, + "baseline_applicable": 0, + "baseline_not_applicable": 0, + "not_protected_in_parent": 0, # U_t items (C-210), also counted under "invariant" + } + + +def _work( + report: VerificationReport, + impact: ImpactReport, + selected: Sequence[VerificationWorkItem] | None, +) -> tuple[list[dict[str, Any]], dict[str, int]]: + listed = {item.target_id for item in impact.verification_work} + reasons = {item.target_id: item.reason.value for item in selected or ()} + rows = [] + counts = _counts() + for r in sorted(report.results, key=lambda x: x.target_id): + kind = r.target_type.value + if kind == GATE_TYPE: + origin = "gate" + counts["gate"] += 1 + else: + origin = "impact" if r.target_id in listed else "condition" + if kind == "INVARIANT": + counts["invariant"] += 1 + if reasons.get(r.target_id) == NOT_PROTECTED_IN_PARENT: + counts["not_protected_in_parent"] += 1 + elif r.applicable: + counts["baseline_applicable"] += 1 + else: + counts["baseline_not_applicable"] += 1 + rows.append( + { + "target_id": r.target_id, + "target_type": kind, + "class": r.verification_class.value, + "result": r.result.value, + "reason": None if r.reason is None else r.reason.value, + "applicable": r.applicable, + "verifier_id": r.verifier_id, + "verifier_version": r.verifier_version, + "evidence_level": r.evidence_level, + "origin": origin, + "selected_reason": reasons.get(r.target_id), # the work item's reason (C-211) + } + ) + return rows, counts + + +def _required(result: EvaluationResult) -> list[dict[str, Any]]: + decision = result.decision + if decision is None: + return [] + blocking: dict[str, list[str]] = {} + for condition in decision.blocking_conditions: + blocking.setdefault(condition.target, []).append(condition.code.value) + return [ + { + "target_id": check.target_id, + "kind": check.kind, + "source": None if check.source is None else check.source.value, + "verification_id": check.verification_id, + "blocking": sorted(blocking.get(check.target_id, [])), + } + for check in decision.required_checks + ] + + +def decision_summary(result: EvaluationResult) -> dict[str, Any]: + if result.kind is ResultKind.CONSTRUCTION_FAILED or result.decision is None: + return { + "type": "REJECT", + "reason": "CONSTRUCTION_FAILED", + "decision_id": None, + "blocking": [], + "note": "construction failure: DAIL stores no decision (C-170 (3)); counted as REJECT", + } + decision = result.decision + return { + "type": decision.decision.value, + "reason": decision.reason.value, + "decision_id": decision.decision_id, + "blocking": [c.to_dict() for c in decision.blocking_conditions], + "note": None, + } + + +def transition_record( + *, + base: Mapping[str, Any], + step: int, + admission: Mapping[str, Any], + parent: TrustedState, + open_targets: Sequence[str], + result: EvaluationResult, +) -> dict[str, Any]: + """The record of one evaluated transition (see the module docstring).""" + impact, report, identity = result.impact, result.verification, result.identity + work: list[dict[str, Any]] = [] + counts = _counts() + if report is not None and impact is not None: + work, counts = _work(report, impact, result.work) + return { + **base, + "record": "transition", + "step_index": step, + "status": EVALUATED, + "admission": dict(admission), + "parent_state_id": parent.state_id, + "parent_state_hash": parent.state_hash, + "candidate_id": result.candidate.candidate_id, + "candidate_state_hash": result.candidate.state_hash, + "open_targets": list(open_targets), + "outcome_class": DAIL_OUTCOME, + "result_kind": result.kind.value, + "decision": decision_summary(result), + "required": _required(result), + "work": work, + "work_counts": counts, + "identity": ( + None + if identity is None + else { + "algorithm_version": identity.algorithm_version, + "content_hash": identity.content_hash, + "pairs": [ + { + "trusted_ref": m.trusted_resource_ref, + "candidate_ref": m.candidate_resource_ref, + "change_type": m.change_type.value, + "moved": m.moved, + } + for m in identity.resource_matches + ], + } + ), + "impact": ( + None + if impact is None + else { + "algorithm_version": impact.algorithm_version, + "content_hash": impact.content_hash, + "affected": [ + { + "invariant_id": a.invariant_id, + "invariant_version": a.invariant_version, + "status": a.status.value, + "impact_types": [t.value for t in a.impact_types], + } + for a in impact.affected_invariants + ], + } + ), + "new_state": None if result.state is None else state_summary(result.state), + "registry": dict(REGISTRY_RECORD), + } + + +def truncated_record(*, base: Mapping[str, Any], step: int) -> dict[str, Any]: + """A step that was not evaluated because an earlier one was not a PROMOTE (C-200).""" + return { + **base, + "record": "transition", + "step_index": step, + "status": TRUNCATED, + "admission": None, + "parent_state_id": None, + "parent_state_hash": None, + "candidate_id": None, + "candidate_state_hash": None, + "open_targets": [], + "outcome_class": NOT_EVALUATED, + "result_kind": TRUNCATED, + "decision": None, + "required": [], + "work": [], + "work_counts": _counts(), + "identity": None, + "impact": None, + "new_state": None, + "registry": dict(REGISTRY_RECORD), + } + + +def work_items( + items: Sequence[VerificationWorkItem] | None, impact: ImpactReport | None +) -> list[Any]: + """The work the scope policy selected, with its origin, for the verification artifact.""" + if items is None: + return [] + listed = set() if impact is None else {w.target_id for w in impact.verification_work} + return [ + {**item.to_dict(), "origin": "impact" if item.target_id in listed else "condition"} + for item in items + ] diff --git a/experiments/runner.py b/experiments/runner.py new file mode 100644 index 0000000..5c5679e --- /dev/null +++ b/experiments/runner.py @@ -0,0 +1,441 @@ +"""Plan, run and resume an experiment (P8a; Doc 13 §5-§7, §39, §43-§48, §50; C-200 to C-207). + +``plan`` turns an ``ExperimentSpec`` and a verified ``LoadedSet`` into an ``Experiment``: ids, +fixture and baseline sets, the planned trials (every target scenario under every condition, +repetition 1) and the pins of §46 (DAIL commit, TerraPreserve tag and manifest hash, registry, +policy, scope, identity and verifier versions, Python version and platform). + +``run_experiment`` writes the definition and the fixture hashes once (a stored definition that +differs is refused: a change needs a new experiment version, §39, §48), then runs every planned +trial that is not finished: + +* a trial whose last event is COMPLETED or FAILED is skipped (a FAILED trial is not retried here; + its error is in its record); +* a trial left INITIALIZED or EXECUTING by an interrupted run is marked ABORTED and re-run as a new + attempt in a new directory: nothing stored is overwritten (§47); +* a PLANNED (or ABORTED) trial runs its next attempt. + +Failure classes (§43-§44): an ``AdmissionError`` is an INPUT_FAILURE and any other ``Exception`` a +HARNESS_FAILURE; either marks the trial FAILED with the error and never becomes a decision. A +``BaseException`` that is not an ``Exception`` (the process is being stopped) is not caught: the +trial stays EXECUTING and the next run resumes it. + +Trials are independent (C-202), so ``workers > 1`` runs them in threads; each has its own store. +""" + +from __future__ import annotations + +import platform +import shutil +import sys +import tempfile +import time +from collections.abc import Callable, Iterable, Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final + +from core.application.config import ResolvedConfig, load_config +from core.domain.registry_v1_1 import REGISTRY_TAG_V1_1, REGISTRY_VERSION_V1_1 +from core.identity.strategy import ADDRESS_ONLY_ALGORITHM_VERSION, IDENTITY_ALGORITHM_VERSION +from core.terraform_model.normalize import NORMALIZATION_VERSION +from core.verification.engine import VERIFIER_SET_VERSION +from experiments import records +from experiments.artifacts import ( + ArtifactError, + Layout, + append_event, + read_record, + write_or_same, + write_record, +) +from experiments.artifacts import read_events as _read_events +from experiments.conditions import setup_for +from experiments.definition import ( + TERMINAL, + Condition, + Experiment, + ExperimentSpec, + TrialStatus, + check_transition, + experiment_id_for, + planned_trials, +) +from experiments.loader import LoadedSet, ScenarioInput, sha256_hex +from experiments.trial import ( + DAIL_CLOCK_BASE, + AdmissionError, + TrialInputs, + correlation_id_for, + run_attempt, +) + +REPO_ROOT: Final = Path(__file__).resolve().parents[1] +EXPERIMENT_PROMOTION_FILE: Final = REPO_ROOT / "experiments" / "config" / "promotion.toml" +S = TrialStatus +Hook = Callable[[str, str], None] + +__all__ = [ + "ExperimentError", + "Layout", + "RunSummary", + "experiment_config", + "plan", + "read_events", + "read_result", + "run_experiment", +] + + +class ExperimentError(Exception): + """The experiment cannot run as asked (a changed definition, a mismatch with the input).""" + + +@dataclass(frozen=True) +class RunSummary: + statuses: Mapping[str, TrialStatus] + attempted: tuple[str, ...] + skipped: tuple[str, ...] + failures: Mapping[str, str] = field(default_factory=dict) + + +# --- configuration -------------------------------------------------------------------------------- + + +def experiment_config() -> ResolvedConfig: + """The repository configuration with the reviewed experiment promotion file in place of + ``config/base/promotion.toml`` (C-207), loaded for the ``experiment`` environment with no + environment variables. Refuses to start if automatic promotion is not on.""" + with tempfile.TemporaryDirectory(prefix="dail-experiment-config-") as tmp: + root = Path(tmp) / "config" + shutil.copytree(REPO_ROOT / "config", root) + shutil.copyfile(EXPERIMENT_PROMOTION_FILE, root / "base" / "promotion.toml") + config = load_config(root, "experiment", environ={}) + if config.get("promotion.automatic_enabled") is not True: + raise ExperimentError("the experiment configuration must enable automatic promotion") + return config + + +# --- plan ----------------------------------------------------------------------------------------- + + +def _verifiers(scenario: ScenarioInput) -> dict[str, Any]: + setup = setup_for(Condition.C, scenario, DAIL_CLOCK_BASE) + return { + "verifier_set_version": VERIFIER_SET_VERSION, + "invariants": { + d.invariant_id: { + "version": d.version, + "verifier_id": d.verifier_id, + "verifier_version": d.verifier_version, + } + for d in setup.definitions + }, + } + + +def plan( + spec: ExperimentSpec, + loaded: LoadedSet, + *, + environment: Mapping[str, str], + created_at: datetime, + only: Iterable[str] | None = None, +) -> Experiment: + """The experiment of ``spec`` over ``loaded`` (every target, or the ids in ``only``).""" + by_id = {s.scenario_id: s for s in loaded.scenarios} + chosen = sorted(by_id) if only is None else sorted(set(only)) + missing = [sid for sid in chosen if sid not in by_id] + if missing or not chosen: + raise ExperimentError(f"no such scenario in the loaded set: {missing or 'none chosen'}") + experiment_id = experiment_id_for(spec.name, spec.version) + first = by_id[chosen[0]] + pins: dict[str, Any] = { + "python": sys.version.split()[0], + "platform": platform.platform(), + **dict(environment), + "terrapreserve_manifest_sha256": loaded.manifest_sha256, + "terrapreserve_split": loaded.split, + "invariant_registry": { + "tag": REGISTRY_TAG_V1_1, + "stored_invariant_registry_version": REGISTRY_VERSION_V1_1, + }, + "policies": { + c.value: setup_for(c, first, DAIL_CLOCK_BASE).versions() for c in spec.conditions + }, + "identity": { + "evidence": IDENTITY_ALGORITHM_VERSION, + "address_only": ADDRESS_ONLY_ALGORITHM_VERSION, + }, + "normalization_version": NORMALIZATION_VERSION, + "verifiers": _verifiers(first), + "dail_clock": f"logical: {DAIL_CLOCK_BASE.isoformat()} plus one second per call (C-207)", + "experiment_promotion_config_sha256": sha256_hex(EXPERIMENT_PROMOTION_FILE.read_bytes()), + "admission_evidence": loaded.admission_evidence, + } + fixtures = tuple(f for f in loaded.fixtures if f.scenario_id in set(chosen)) + return Experiment( + experiment_id=experiment_id, + name=spec.name, + version=spec.version, + objective=spec.objective, + hypothesis=spec.hypotheses, + protocol_version=spec.protocol_version, + fixture_set=tuple(chosen), + baseline_set=tuple((sid, by_id[sid].steps[0].content_hash) for sid in chosen), + condition_set=spec.conditions, + repetitions=spec.repetitions, + created_at=created_at, + pins=pins, + fixtures=fixtures, + planned=planned_trials(experiment_id, chosen, spec.conditions), + ) + + +# --- reading back --------------------------------------------------------------------------------- + + +def read_events(layout: Layout, trial_id: str) -> list[dict[str, Any]]: + return _read_events(layout.events_path(trial_id)) + + +def read_result(layout: Layout, trial_id: str) -> dict[str, Any]: + """The result of the trial's latest attempt that has one.""" + for attempt in reversed(layout.attempts(trial_id)): + path = layout.attempt_dir(trial_id, attempt) / "result.json" + if path.exists(): + data = read_record(path) + assert isinstance(data, dict) + return data + raise ExperimentError(f"trial {trial_id} has no stored result") + + +# --- run ------------------------------------------------------------------------------------------ + + +def _definition_without_clock(data: Mapping[str, Any]) -> dict[str, Any]: + return {k: v for k, v in data.items() if k != "created_at"} + + +def _write_definition(layout: Layout, experiment: Experiment) -> None: + wanted = experiment.to_dict() + path = layout.definition_path + if path.exists(): + stored = read_record(path) + if not isinstance(stored, dict) or _definition_without_clock( + stored + ) != _definition_without_clock(wanted): + raise ExperimentError( + f"{path} holds another definition of experiment {experiment.experiment_id}; a " + "changed experiment needs a new version (Doc 13 §39, §48)" + ) + else: + write_record(path, wanted) + try: + write_or_same(layout.fixtures_path, experiment.fixtures_dict()) + except ArtifactError as changed: + raise ExperimentError(f"the fixtures changed: {changed}") from None + + +def _wall() -> str: + return datetime.now(UTC).isoformat() + + +class _TrialRun: + def __init__( + self, + layout: Layout, + experiment: Experiment, + trial_id: str, + scenario: ScenarioInput, + condition: Condition, + repetition: int, + evidence: str, + ) -> None: + self.layout = layout + self.experiment = experiment + self.trial_id = trial_id + self.scenario = scenario + self.condition = condition + self.repetition = repetition + self.evidence = evidence + self.path = layout.events_path(trial_id) + + def event( + self, attempt: int, status: TrialStatus, last: TrialStatus | None, **extra: Any + ) -> None: + if last is not None: + check_transition(last, status) + append_event( + self.path, + { + "trial_id": self.trial_id, + "correlation_id": correlation_id_for(self.trial_id), + "attempt": attempt, + "status": status.value, + "at": _wall(), + **extra, + }, + ) + + def state(self) -> tuple[TrialStatus | None, int]: + events = _read_events(self.path) + if not events: + return None, 0 + return TrialStatus(events[-1]["status"]), max(int(e["attempt"]) for e in events) + + def run(self, config: Any, hook: Hook | None) -> tuple[TrialStatus, bool, str | None]: + """(status, attempted, failure message).""" + last, attempt = self.state() + if last is None: + self.event(0, S.PLANNED, None) + last = S.PLANNED + if last in TERMINAL: + return last, False, None + if last in (S.INITIALIZED, S.EXECUTING): # an interrupted run (§50) + self.event(attempt, S.ABORTED, last, reason="interrupted run; re-run as a new attempt") + last = S.ABORTED + number = max(attempt, *self.layout.attempts(self.trial_id), 0) + 1 + directory = self.layout.attempt_dir(self.trial_id, number) + if directory.exists(): + raise ExperimentError(f"{directory} exists; a stored attempt is never overwritten") + self.event(number, S.INITIALIZED, last) + directory.mkdir(parents=True) + started_wall, started = _wall(), time.perf_counter() + if hook is not None: + hook(self.trial_id, "initialized") + self.event(number, S.EXECUTING, S.INITIALIZED) + inputs = TrialInputs( + trial_id=self.trial_id, + experiment_id=self.experiment.experiment_id, + scenario=self.scenario, + condition=self.condition, + repetition=self.repetition, + attempt=number, + admission_evidence=self.evidence, + ) + failure: dict[str, Any] | None = None + durations: dict[str, float] = {} + try: + outcome = run_attempt(inputs, directory, config=config, hook=hook) + result, durations = outcome.result, outcome.durations + except Exception as error: # classified; never a decision (§43-§44) + failure = { + "class": ( + records.INPUT_FAILURE + if isinstance(error, AdmissionError) + else records.HARNESS_FAILURE + ), + "type": type(error).__name__, + "message": str(error)[:2000], + } + result = { + "record": "trial_result", + "trial_id": self.trial_id, + "correlation_id": correlation_id_for(self.trial_id), + "experiment_id": self.experiment.experiment_id, + "condition": self.condition.value, + "scenario_id": self.scenario.scenario_id, + "family": self.scenario.family, + "repetition": self.repetition, + "attempt": number, + "status": "FAILED", + "failure": failure, + "baseline": None, + "transitions": [], + } + status = S.FAILED if failure is not None else S.COMPLETED + elapsed = time.perf_counter() - started + write_record(directory / "result.json", result) + baseline = result.get("baseline") or {} + write_record( + directory / "timing.json", + { + "trial": { + "trial_id": self.trial_id, + "experiment_id": self.experiment.experiment_id, + "condition_id": self.condition.value, + "fixture_id": self.scenario.scenario_id, + "baseline_id": baseline.get("baseline_id"), + "repetition": self.repetition, + "seed": None, + "parent_state_id": baseline.get("state_id"), + "status": status.value, + "started_at": started_wall, + "completed_at": _wall(), + "artifact_refs": sorted( + p.relative_to(directory).as_posix() + for p in directory.rglob("*.json") + if p.name != "timing.json" + ), + "attempt": number, + }, + "durations_seconds": {"total": elapsed, **durations}, + }, + ) + self.event(number, status, S.EXECUTING) + message = None if failure is None else f"{failure['type']}: {failure['message']}" + return status, True, message + + +def run_experiment( + experiment: Experiment, + loaded: LoadedSet, + output_root: Path, + *, + hook: Hook | None = None, + workers: int = 1, + config: Any = None, +) -> RunSummary: + """Run every unfinished planned trial of ``experiment`` (see the module docstring).""" + by_id = {s.scenario_id: s for s in loaded.scenarios} + stored = {f.scenario_id: f for f in loaded.fixtures} + for fixture in experiment.fixtures: + if stored.get(fixture.scenario_id) != fixture: + raise ExperimentError( + f"{fixture.scenario_id}: the loaded fixture is not the experiment's" + ) + layout = Layout(Path(output_root), experiment.experiment_id) + _write_definition(layout, experiment) + chosen_config = config if config is not None else experiment_config() + runs = [ + _TrialRun( + layout, + experiment, + t.trial_id, + by_id[t.scenario_id], + t.condition, + t.repetition, + loaded.admission_evidence, + ) + for t in experiment.planned + ] + statuses: dict[str, TrialStatus] = {} + attempted: list[str] = [] + skipped: list[str] = [] + failures: dict[str, str] = {} + + def collect(item: _TrialRun, outcome: tuple[TrialStatus, bool, str | None]) -> None: + status, ran, message = outcome + statuses[item.trial_id] = status + (attempted if ran else skipped).append(item.trial_id) + if message is not None: + failures[item.trial_id] = message + + if workers <= 1: + for item in runs: + collect(item, item.run(chosen_config, hook)) + else: + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [(item, pool.submit(item.run, chosen_config, hook)) for item in runs] + for item, future in futures: + collect(item, future.result()) + order = {t.trial_id: n for n, t in enumerate(experiment.planned)} + return RunSummary( + statuses=dict(sorted(statuses.items(), key=lambda kv: order[kv[0]])), + attempted=tuple(sorted(attempted, key=order.__getitem__)), + skipped=tuple(sorted(skipped, key=order.__getitem__)), + failures=failures, + ) diff --git a/experiments/trial.py b/experiments/trial.py new file mode 100644 index 0000000..f0749f9 --- /dev/null +++ b/experiments/trial.py @@ -0,0 +1,365 @@ +"""One attempt of one trial (P8a; Doc 13 Appendix A, §13, §20, §40; C-196, C-200 to C-203). + + initialize: a fresh store (its own SQLite file), a fresh lineage (C-202) + baseline: S0 through the existing baseline flow (P6c) with the scenario's bindings and primary + variable values; DAIL's own S0 results are recorded as they are + for step k = 1..n: admit the step (its content hash, C-201), evaluate it as a FIXED_PATCH + candidate against the trial's current Trusted State through the real + ``EvaluationOrchestrator``; stop at the first decision that is not PROMOTE and + record the rest as TRUNCATED (C-200) + +Nothing here decides anything: every transition is one ``evaluate`` call, and the stores of two +trials have nothing in common but the read-only scenario files. No experiment lineage is ever the +authoritative lineage (Doc 13 §20): the store lives under the attempt's own directory. + +Time (C-207): everything DAIL hashes uses a logical clock that starts at ``DAIL_CLOCK_BASE`` and +moves one second per call, so a re-run of the same trial stores the same records. The wall clock is +kept apart in ``timing.json``. Ids are name-based from the trial id and the attempt number. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, Final + +from core.application.baseline import BaselineRefused, BaselineRequest, establish_baseline +from core.application.orchestrator import EvaluationOrchestrator, EvaluationRequest +from core.domain.enums import CandidateSource, DecisionType +from core.domain.patch import Patch +from core.domain.registry_v1_1 import REGISTRY_VERSION_V1_1 +from core.domain.repositories import UnitOfWork +from core.domain.state import TrustedState +from core.persistence.schema import initialize_database +from core.persistence.sqlite import SqliteUnitOfWork +from core.promotion.commit import PromotionContext +from experiments import records +from experiments.artifacts import ATTEMPT_DIRS, write_record +from experiments.conditions import REGISTRY_RECORD, ConditionSetup, setup_for +from experiments.definition import Condition, name_id +from experiments.loader import ScenarioInput, StepInput, step_content_hash + +DAIL_CLOCK_BASE: Final = datetime(2026, 1, 1, tzinfo=UTC) +STORE_NAME: Final = "dail.sqlite" +Hook = Callable[[str, str], None] + + +class AdmissionError(Exception): + """A step's content hash is not the verified one (an input failure, C-201).""" + + +class LogicalClock: + """``DAIL_CLOCK_BASE`` plus one second per call (C-207).""" + + def __init__(self) -> None: + self.calls = 0 + + def __call__(self) -> datetime: + self.calls += 1 + return DAIL_CLOCK_BASE + timedelta(seconds=self.calls) + + +@dataclass(frozen=True) +class TrialInputs: + trial_id: str + experiment_id: str + scenario: ScenarioInput + condition: Condition + repetition: int + attempt: int + admission_evidence: str + + +@dataclass +class AttemptOutcome: + """What an attempt produced: the result record (no wall clock) and the durations.""" + + result: dict[str, Any] + durations: dict[str, float] + + +def correlation_id_for(trial_id: str) -> str: + return name_id("correlation", trial_id) + + +def _admit(step: StepInput, evidence: str) -> dict[str, Any]: + recomputed = step_content_hash(step.directory) + if recomputed != step.content_hash: + raise AdmissionError( + f"step {step.index} ({step.directory}): content hash {recomputed} is not the " + f"manifest's {step.content_hash}" + ) + return { + "step": step.index, + "content_hash": recomputed, + "content_hash_matches_manifest": True, + "terraform_validate": evidence, + } + + +class _Attempt: + def __init__(self, inputs: TrialInputs, attempt_dir: Path, config: Any, hook: Hook | None): + self.i = inputs + self.dir = attempt_dir + self.config = config + self.hook = hook + self.clock = LogicalClock() + self.correlation_id = correlation_id_for(inputs.trial_id) + self.lineage_id = self.id("lineage") + self.durations: dict[str, float] = {} + self.base: dict[str, Any] = { + "trial_id": inputs.trial_id, + "correlation_id": self.correlation_id, + "condition": inputs.condition.value, + "scenario_id": inputs.scenario.scenario_id, + } + + def id(self, *parts: object) -> str: + return name_id("trial-attempt", self.i.trial_id, self.i.attempt, *parts) + + def at(self, stage: str) -> None: + if self.hook is not None: + self.hook(self.i.trial_id, stage) + + def context(self, step: int) -> PromotionContext: + return PromotionContext( + run_id=self.id("run"), + correlation_id=self.correlation_id, + operation_id=self.id("operation", step), + attempt_id=self.id("attempt"), + actor_id=None, + ) + + def write(self, folder: str, name: str, data: Mapping[str, Any]) -> None: + write_record(self.dir / folder / name, {**self.base, **data}) + + # --- the attempt -------------------------------------------------------------------------- + + def run(self) -> AttemptOutcome: + scenario = self.i.scenario + setup = setup_for(self.i.condition, scenario, DAIL_CLOCK_BASE) + for name in ATTEMPT_DIRS: + (self.dir / name).mkdir(parents=True, exist_ok=True) + store = self.dir / "store" / STORE_NAME + store.parent.mkdir(parents=True, exist_ok=True) + initialize_database(store) + + def factory() -> UnitOfWork: + return SqliteUnitOfWork(store) + + result: dict[str, Any] = { + **self.base, + "record": "trial_result", + "experiment_id": self.i.experiment_id, + "family": scenario.family, + "repetition": self.i.repetition, + "attempt": self.i.attempt, + "lineage_id": self.lineage_id, + "versions": setup.versions(), + "registry": dict(REGISTRY_RECORD), + "protected_invariants": setup.protected_mapping, + "status": "COMPLETED", + "failure": None, + "baseline": None, + "transitions": [], + } + self.at("baseline") + started = time.perf_counter() + parent = self._baseline(factory, setup, result) + self.durations["baseline"] = time.perf_counter() - started + if parent is None: + result["transitions"] = [ + records.truncated_record(base=self.base, step=k) + for k in range(1, scenario.transitions + 1) + ] + return AttemptOutcome(result, self.durations) + orchestrator = EvaluationOrchestrator( + factory, + config=self.config, + scope_policy=setup.scope_policy, + clock=self.clock, + policy_version=setup.policy_version, + policy_factory=setup.policy_factory, + identity_strategy=setup.identity_strategy, + ) + transitions: list[dict[str, Any]] = result["transitions"] + for step in scenario.steps[1:]: + self.at(f"step-{step.index}") + started = time.perf_counter() + state, record = self._step(orchestrator, setup, parent, step) + self.durations[f"step-{step.index}"] = time.perf_counter() - started + transitions.append(record) + if state is None: + transitions.extend( + records.truncated_record(base=self.base, step=later.index) + for later in scenario.steps[step.index + 1 :] + ) + break + parent = state + return AttemptOutcome(result, self.durations) + + def _baseline( + self, factory: Callable[[], UnitOfWork], setup: ConditionSetup, result: dict[str, Any] + ) -> TrustedState | None: + scenario = self.i.scenario + admission = _admit(scenario.steps[0], self.i.admission_evidence) + with factory() as uow: # earlier versions first: no gaps (Doc 06 §4.2, C-203) + for definition in setup.history: + uow.invariants.register_definition(definition) + request = BaselineRequest( + baseline_id=self.id("baseline"), + lineage_id=self.lineage_id, + baseline_dir=scenario.steps[0].directory, + variables=dict(scenario.variable_values), + definitions=setup.baseline_definitions, + bindings=setup.baseline_bindings, + context=self.context(0), + now=self.clock(), + eligible=True, + registry_version=REGISTRY_VERSION_V1_1, + allow_no_invariants=setup.allow_no_invariants, + ) + try: + made = establish_baseline(factory, request) + except BaselineRefused as refused: + summary: dict[str, Any] = { + "outcome": "REFUSED", + "reasons": list(refused.reasons), + "admission": admission, + "invariants": {}, + "state_id": None, + "state_hash": None, + } + result["baseline"] = summary + self.write("baseline", "baseline.json", {"record": "baseline", **summary}) + return None + summary = { + "outcome": "ESTABLISHED", + "reasons": [], + "admission": admission, + **records.state_summary(made.state), + "baseline_id": request.baseline_id, + "registry": dict(REGISTRY_RECORD), + } + result["baseline"] = summary + self.write( + "baseline", + "baseline.json", + { + "record": "baseline", + **summary, + "verification": records.stable(made.report.to_dict()), + }, + ) + return made.state + + def _step( + self, + orchestrator: EvaluationOrchestrator, + setup: ConditionSetup, + parent: TrustedState, + step: StepInput, + ) -> tuple[TrustedState | None, dict[str, Any]]: + scenario = self.i.scenario + admission = _admit(step, self.i.admission_evidence) + opened = setup.open_targets(scenario, step.index) + patch = Patch.create( + source=CandidateSource.FIXED_PATCH, + content=( + f"TerraPreserve {scenario.scenario_id} step {step.index} " + f"({step.content_hash})\n" + ), + parent_state_id=parent.state_id, + now=self.clock(), + patch_id=self.id("patch", step.index), + ) + request = EvaluationRequest( + lineage_id=self.lineage_id, + candidate_id=self.id("candidate", step.index), + candidate_sequence=step.index, + attempt_number=1, + patch=patch, + candidate_dir=step.directory, + variables=dict(scenario.variable_values), + definitions=setup.definitions, + bindings=setup.bindings, + context=self.context(step.index), + open_targets=opened, + ) + evaluated = orchestrator.evaluate(request) + record = records.transition_record( + base=self.base, + step=step.index, + admission=admission, + parent=parent, + open_targets=opened, + result=evaluated, + ) + name = f"step-{step.index}.json" + self.write( + "generation", + name, + {"record": "fixed_patch", "patch": patch.to_dict(), "admission": admission}, + ) + self.write( + "analysis", + name, + { + "record": "analysis", + "identity": ( + None + if evaluated.identity is None + else records.stable(evaluated.identity.to_dict()) + ), + "impact": ( + None if evaluated.impact is None else records.stable(evaluated.impact.to_dict()) + ), + "registry": dict(REGISTRY_RECORD), + }, + ) + self.write( + "verification", + name, + { + "record": "verification", + "work": records.work_items(evaluated.work, evaluated.impact), + "report": ( + None + if evaluated.verification is None + else records.stable(evaluated.verification.to_dict()) + ), + "registry": dict(REGISTRY_RECORD), + }, + ) + self.write( + "decision", + name, + { + "transition": record, + "decision": None if evaluated.decision is None else evaluated.decision.to_dict(), + "evidence": ( + None if evaluated.outcome is None else evaluated.outcome.evidence.to_dict() + ), + }, + ) + promoted = ( + evaluated.decision is not None + and evaluated.decision.decision is DecisionType.PROMOTE + and evaluated.state is not None + ) + return (evaluated.state if promoted else None), record + + +def run_attempt( + inputs: TrialInputs, attempt_dir: Path, *, config: Any, hook: Hook | None = None +) -> AttemptOutcome: + """Run one attempt (see the module docstring). Exceptions propagate to the runner, which + classifies them; nothing here turns an exception into a decision.""" + return _Attempt(inputs, attempt_dir, config, hook).run() + + +def setup_versions(condition: Condition, scenario: ScenarioInput) -> dict[str, Any]: + return setup_for(condition, scenario, DAIL_CLOCK_BASE).versions() diff --git a/pyproject.toml b/pyproject.toml index 9a37429..1c86107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ strict = true warn_unused_ignores = true disallow_untyped_defs = true # Tests are not type-checked. -files = ["core", "evidence", "llm", "scripts"] +files = ["core", "evidence", "llm", "scripts", "experiments"] # LLM/AWS-adapter modules get relaxed typing rules once they exist (P7/P9), # since third-party stubs for litellm/boto3 are inconsistent; core.domain and # core.identity/dependency/impact/verification/promotion must stay strict. diff --git a/tests/contract/test_experiment_blinding.py b/tests/contract/test_experiment_blinding.py new file mode 100644 index 0000000..b558531 --- /dev/null +++ b/tests/contract/test_experiment_blinding.py @@ -0,0 +1,113 @@ +"""Label blinding of the experiment harness (P8a; work package §4.2, "Blinding"). + +"Trial execution never reads oracle labels. Labels are used only for scoring, which is P8b." +§4.2: "Gives execution only: scenario_id, family, split, template_id or lineage id, the step +directories and hashes, target.kind, target.invariant_id, protected_invariants, endpoints (for the +INV-FUNC-001 bindings) and the primary variable_values. Never labels, gt_class, oracle, +mutation_record, intent, variable_values_alt or anything else. A contract test fails if any module +used by trial execution reads one of those fields (for example by parsing the sources with ast)." + +Every module of ``experiments/`` is used by trial execution in P8a (scoring is P8b, and a scoring +module would be listed here by name as the one exception). The check parses each source with ``ast`` +and reports any use of a forbidden field name: a string constant that is not a docstring (a key, a +``get`` or ``getattr`` argument, a set or list member), an attribute, a name or a keyword argument. +A self-test feeds sources that read each field, so a checker that can no longer fail is caught. +""" + +from __future__ import annotations + +import ast +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +EXPERIMENTS = REPO_ROOT / "experiments" +FORBIDDEN = frozenset( + {"labels", "gt_class", "oracle", "mutation_record", "intent", "variable_values_alt"} +) +# P8b's scoring module(s) will be the only exception; none exists in P8a. +SCORING_MODULES: frozenset[str] = frozenset() + + +def _docstrings(tree: ast.AST) -> set[int]: + """The ids of the constant nodes that are docstrings or other bare string statements.""" + found: set[int] = set() + for node in ast.walk(tree): + if ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + found.add(id(node.value)) + return found + + +def label_reads(source: str) -> list[str]: + """Every use of a forbidden field name in ``source``, as 'line: name'.""" + tree = ast.parse(source) + skip = _docstrings(tree) + found: list[str] = [] + for node in ast.walk(tree): + name: str | None = None + if isinstance(node, ast.Constant) and isinstance(node.value, str) and id(node) not in skip: + name = node.value if node.value in FORBIDDEN else None + elif isinstance(node, ast.Attribute) and node.attr in FORBIDDEN: + name = node.attr + elif isinstance(node, ast.Name) and node.id in FORBIDDEN: + name = node.id + elif isinstance(node, ast.keyword) and node.arg in FORBIDDEN: + name = node.arg + if name is not None: + found.append(f"{getattr(node, 'lineno', '?')}: {name}") + return found + + +def execution_modules() -> list[Path]: + return sorted( + p + for p in EXPERIMENTS.rglob("*.py") + if "__pycache__" not in p.parts and p.stem not in SCORING_MODULES + ) + + +class TestBlinding(unittest.TestCase): + def test_no_execution_module_reads_a_label_field(self) -> None: + modules = execution_modules() + self.assertTrue(modules, "experiments/ has no module to check") + self.assertIn(EXPERIMENTS / "loader.py", modules) + for path in modules: + with self.subTest(module=path.relative_to(REPO_ROOT).as_posix()): + self.assertEqual(label_reads(path.read_text(encoding="utf-8")), []) + + def test_no_scoring_module_exists_yet(self) -> None: + self.assertEqual(SCORING_MODULES, frozenset()) + + +class TestTheCheckerCanFail(unittest.TestCase): + def test_each_kind_of_read_is_found(self) -> None: + for source in ( + 'value = data["labels"]\n', + 'value = data.get("gt_class")\n', + 'value = getattr(scenario, "oracle")\n', + "value = scenario.intent\n", + 'wanted = {"mutation_record"}\n', + 'keys = ["scenario_id", "variable_values_alt"]\n', + "def f(labels):\n return labels\n", + "call(oracle=1)\n", + 'value = data[f"{prefix}"] or data["labels"]\n', + ): + with self.subTest(source=source.strip()): + self.assertNotEqual(label_reads(source), []) + + def test_docstrings_and_allowed_fields_are_not_findings(self) -> None: + source = ( + '"""This module never reads labels, gt_class or the oracle."""\n' + "def f(data):\n" + ' """Not the intent."""\n' + ' return data["variable_values"], data["scenario_id"], data["target"]\n' + ) + self.assertEqual(label_reads(source), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/experiment_builders.py b/tests/experiment_builders.py new file mode 100644 index 0000000..b95f9fb --- /dev/null +++ b/tests/experiment_builders.py @@ -0,0 +1,180 @@ +"""Builders for the experiment-harness tests (P8a). Not a test module. + +``mini_root`` writes a small TerraPreserve-layout tree into a temporary directory from the spec in +``tests/fixtures/terrapreserve_mini/scenarios.json``: ``scenarios/V1_MANIFEST.json`` and +``scenarios/dev//{scenario.json, step_0/, step_1/, ...}``. Step directories are copied +from DAIL's canonical fixtures (``main.tf`` and ``versions.tf`` only) with the spec's text edits. + +The hashes are computed here, from the definition the work package quotes (D4-09): a step's +``content_hash`` is the SHA-256 of the newline-joined, path-sorted lines +``":"``; ``scenario_json_sha256`` is the SHA-256 of +the file's bytes; the manifest pin is the SHA-256 of the manifest's bytes. Nothing is copied from +TerraPreserve. + +The manifest also has a ``test`` list whose entry points at a directory that does not exist: a +loader that read the test split would fail. Label fields are filled with junk values; ``corrupt`` +changes every one of them, for the blinding test. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from tests.fixture_catalog import REPO_ROOT + +SPEC = REPO_ROOT / "tests" / "fixtures" / "terrapreserve_mini" / "scenarios.json" +COPIED = ("main.tf", "versions.tf") +F2, F8, F1, F4, RESERVE, F8_BROKEN, F8_UNTOUCHED = ( + "tp-mini-F2-0001", + "tp-mini-F8-0002", + "tp-mini-F1-0003", + "tp-mini-F4-0004", + "tp-mini-F2-0005", + "tp-mini-F8-0006", + "tp-mini-F8-0007", +) +TARGETS = (F1, F2, F4, F8, F8_BROKEN, F8_UNTOUCHED) + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def content_hash(directory: Path) -> str: + """D4-09, written from the quoted definition (independently of experiments.loader).""" + files = sorted( + (p.relative_to(directory).as_posix(), p) for p in directory.rglob("*") if p.is_file() + ) + lines = [f"{rel}:{sha256(path.read_bytes())}" for rel, path in files] + return sha256("\n".join(lines).encode("utf-8")) + + +def _labels(scenario: dict[str, Any], marker: str) -> dict[str, Any]: + """Junk values in every field execution must never read.""" + steps = len(scenario["steps"]) + return { + "labels": [ + { + "step": k, + "invariant_truth": {"INV-SEC-001": marker, "INV-FUNC-001": marker}, + "oracle_affected": [marker], + "target_safe": marker, + } + for k in range(steps) + ], + "gt_class": f"gt-{marker}", + "oracle": {"versions": {"oracle": marker}, "disagreements": [marker]}, + "mutation_record": [{"step": 1, "operator": marker}], + "intent": {"class": marker, "breaks": [marker], "preserves": [marker]}, + "variable_values_alt": {"db_password": f"alt-{marker}"}, + } + + +@dataclass(frozen=True) +class MiniRoot: + root: Path + manifest_sha256: str + + +def mini_root( + directory: Path, + *, + corrupt: bool = False, + edit: Callable[[dict[str, Any]], None] | None = None, +) -> MiniRoot: + """Write the mini TerraPreserve tree under ``directory`` and return its root and manifest pin. + ``edit`` may change each scenario dict before it is written (and hashed).""" + spec = json.loads(SPEC.read_text(encoding="utf-8")) + dev = directory / "scenarios" / "dev" + entries = [] + for item in spec["scenarios"]: + sid = item["scenario_id"] + base = dev / sid + steps = [] + for k, step in enumerate(item["steps"]): + target = base / f"step_{k}" + target.mkdir(parents=True) + for name in COPIED: + text = (REPO_ROOT / step["from"] / name).read_text(encoding="utf-8") + for old, new in step["edits"]: + if name == "main.tf": + assert text.count(old) == 1, (sid, k, old) + text = text.replace(old, new) + (target / name).write_bytes(text.replace("\r\n", "\n").encode("utf-8")) + steps.append( + { + "step": k, + "terraform_dir": f"step_{k}", + "content_hash": content_hash(target), + "diff_from_previous": None, + } + ) + scenario: dict[str, Any] = { + "dataset_version": "1.0", + "scenario_id": sid, + "family": item["family"], + "split": "dev", + "template_id": item["template_id"], + "v1_role": item["v1_role"], + "target": item["target"], + "protected_invariants": [ + {"invariant_id": i, "version": "1.0"} + for i in ("INV-SEC-001", "INV-SEC-002", "INV-SEC-003", "INV-FUNC-001") + ], + "endpoints": spec["endpoints"], + "variable_values": {}, + "steps": steps, + **_labels(item, "corrupted" if corrupt else "junk"), + } + if edit is not None: + edit(scenario) + raw = json.dumps(scenario, indent=1, sort_keys=True).encode("utf-8") + (base / "scenario.json").write_bytes(raw) + entries.append( + { + "scenario_id": sid, + "scenario_json_sha256": sha256(raw), + "steps": [{"step": s["step"], "content_hash": s["content_hash"]} for s in steps], + "v1_role": item["v1_role"], + } + ) + manifest = { + "dataset_version": "1.0", + "decision": "P8a test data", + "dev": entries, + "test": [ + { + "scenario_id": "tp-mini-T-9999", + "scenario_json_sha256": "0" * 64, + "steps": [{"step": 0, "content_hash": "0" * 64}], + "v1_role": "target", + } + ], + } + raw = json.dumps(manifest, indent=1, sort_keys=True).encode("utf-8") + (directory / "scenarios" / "V1_MANIFEST.json").write_bytes(raw) + return MiniRoot(directory, sha256(raw)) + + +class TempDirs: + """Temporary directories removed at cleanup.""" + + def __init__(self) -> None: + self._made: list[tempfile.TemporaryDirectory[str]] = [] + + def new(self, prefix: str = "dail-p8a-") -> Path: + made = tempfile.TemporaryDirectory(prefix=prefix) + self._made.append(made) + return Path(made.name) + + def close(self) -> None: + for made in self._made: + shutil.rmtree(made.name, ignore_errors=True) + made.cleanup() diff --git a/tests/fixtures/terrapreserve_mini/scenarios.json b/tests/fixtures/terrapreserve_mini/scenarios.json new file mode 100644 index 0000000..8694719 --- /dev/null +++ b/tests/fixtures/terrapreserve_mini/scenarios.json @@ -0,0 +1,86 @@ +{ + "_comment": "P8a test data: small scenarios in TerraPreserve's layout, built at test time from DAIL's canonical fixtures (fixtures/canonical_regression). Hashes are computed by tests/experiment_builders.py, never copied from TerraPreserve. Label fields are junk so that a test can corrupt them.", + "endpoints": {"S": "aws_instance.app", "D": "aws_db_instance.database"}, + "scenarios": [ + { + "scenario_id": "tp-mini-F2-0001", + "family": "F2", + "template_id": "tpl-mini", + "v1_role": "target", + "target": {"kind": "repair", "invariant_id": "INV-SEC-001", "description": "Close public SSH."}, + "steps": [ + {"from": "fixtures/canonical_regression/baseline", "edits": []}, + {"from": "fixtures/canonical_regression/patch_safe", "edits": []} + ] + }, + { + "scenario_id": "tp-mini-F8-0002", + "family": "F8", + "template_id": "tpl-mini", + "v1_role": "target", + "target": {"kind": "repair", "invariant_id": "INV-SEC-001", "description": "Close public SSH in two steps."}, + "steps": [ + {"from": "fixtures/canonical_regression/baseline", "edits": []}, + {"from": "fixtures/canonical_regression/baseline", "edits": [["description = \"Application security group\"", "description = \"Application security group, step 1\""]]}, + {"from": "fixtures/canonical_regression/patch_safe", "edits": []} + ] + }, + { + "scenario_id": "tp-mini-F1-0003", + "family": "F1", + "template_id": "tpl-mini", + "v1_role": "target", + "target": {"kind": "change_request", "invariant_id": null, "description": "Resize the application instance."}, + "steps": [ + {"from": "fixtures/canonical_regression/patch_safe", "edits": []}, + {"from": "fixtures/canonical_regression/patch_safe", "edits": [["instance_type = \"t3.micro\"", "instance_type = \"t3.small\""]]} + ] + }, + { + "scenario_id": "tp-mini-F4-0004", + "family": "F4", + "template_id": "tpl-mini", + "v1_role": "target", + "target": {"kind": "repair", "invariant_id": "INV-SEC-001", "description": "Close public SSH (the repair breaks EC2 to RDS)."}, + "steps": [ + {"from": "fixtures/canonical_regression/baseline", "edits": []}, + {"from": "fixtures/canonical_regression/patch_regression", "edits": []} + ] + }, + { + "scenario_id": "tp-mini-F8-0006", + "family": "F8", + "template_id": "tpl-mini", + "v1_role": "target", + "target": {"kind": "repair", "invariant_id": "INV-SEC-001", "description": "Close public SSH in two steps; the first breaks EC2 to RDS."}, + "steps": [ + {"from": "fixtures/canonical_regression/baseline", "edits": []}, + {"from": "fixtures/canonical_regression/patch_regression", "edits": []}, + {"from": "fixtures/canonical_regression/patch_safe", "edits": []} + ] + }, + { + "scenario_id": "tp-mini-F8-0007", + "family": "F8", + "template_id": "tpl-mini", + "v1_role": "target", + "target": {"kind": "repair", "invariant_id": "INV-SEC-001", "description": "Close public SSH in two steps; the first does not touch the security groups."}, + "steps": [ + {"from": "fixtures/canonical_regression/baseline", "edits": []}, + {"from": "fixtures/canonical_regression/baseline", "edits": [["instance_type = \"t3.micro\"", "instance_type = \"t3.small\""]]}, + {"from": "fixtures/canonical_regression/patch_safe", "edits": []} + ] + }, + { + "scenario_id": "tp-mini-F2-0005", + "family": "F2", + "template_id": "tpl-mini", + "v1_role": "reserve", + "target": {"kind": "repair", "invariant_id": "INV-SEC-001", "description": "A reserve: never a trial."}, + "steps": [ + {"from": "fixtures/canonical_regression/baseline", "edits": []}, + {"from": "fixtures/canonical_regression/patch_safe", "edits": []} + ] + } + ] +} diff --git a/tests/unit/test_experiment_harness.py b/tests/unit/test_experiment_harness.py new file mode 100644 index 0000000..5b38bbb --- /dev/null +++ b/tests/unit/test_experiment_harness.py @@ -0,0 +1,558 @@ +"""The experiment harness: conditions, trials, records, artifacts (P8a; Doc 13 §5-§7, §39-§48, §50, +Appendix A; C-196 to C-206). + +Expected values come from the quoted rules, not from a run: + +* the conditions (C-198): A requires the gate items and the repair target only (gate items only for + a change request); B and C require every invariant on the state; C' is C with address-only + identity. The canonical regression (SSH closed, EC2 -> RDS broken) is therefore REJECTED by B, C + and C' and PROMOTED by A, which does not require INV-FUNC-001; the safe repair is PROMOTED by + all four. +* open targets (C-196): on F8 every step but the last has the repair target as an open target, so a + middle step that leaves SSH open is PROMOTED with INV-SEC-001 VIOLATED, and the last step + must PASS. +* trajectory (C-200): "After any decision other than PROMOTE at step i < n, steps i+1..n are not + evaluated and are recorded as TRUNCATED." +* §4.3: harness failure "marks the trial FAILED with the error; it never becomes a decision"; + "re-running the same experiment skips COMPLETED trials, re-runs an unfinished one as a new + attempt, and never overwrites a stored record"; "running one trial twice gives identical records + apart from ids, timestamps and durations". +* §4.2: "a test feeds a scenario whose label fields are corrupted and shows that every trial record + is byte-identical." +* Doc 13 §7: "PLANNED -> INITIALIZED -> EXECUTING -> (FAILED | ABORTED | COMPLETED) -> ANALYZED -> + INCLUDED." §41: "experiment// definition/ fixtures/ trials// { baseline/ + generation/ analysis/ verification/ decision/ metrics/ } report/". +""" + +from __future__ import annotations + +import json +import unittest +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from experiments.definition import ( + Condition, + ExperimentSpec, + TransitionError, + TrialStatus, + check_transition, +) +from experiments.loader import load +from experiments.runner import ( + ExperimentError, + Layout, + plan, + read_events, + read_result, + run_experiment, +) +from tests.experiment_builders import ( + F1, + F2, + F4, + F8, + F8_BROKEN, + F8_UNTOUCHED, + TARGETS, + TempDirs, + mini_root, +) + +A, B, C, CP = Condition.A, Condition.B, Condition.C, Condition.C_PRIME +ALL_CONDITIONS = (A, B, C, CP) +SEC1 = "INV-SEC-001" +CREATED = datetime(2026, 11, 12, 9, 0, tzinfo=UTC) +ENVIRONMENT = { + "dail_commit": "0" * 40, + "dail_worktree_clean": "true", + "terrapreserve_tag": "mini", + "python": "3.12-test", + "platform": "test", +} +# Fields every evaluated transition records (§4.3). +TRANSITION_FIELDS = { + "record", + "trial_id", + "correlation_id", + "condition", + "scenario_id", + "step_index", + "status", + "admission", + "parent_state_id", + "parent_state_hash", + "candidate_id", + "candidate_state_hash", + "open_targets", + "outcome_class", + "result_kind", + "decision", + "required", + "work", + "work_counts", + "identity", + "impact", + "new_state", + "registry", +} + + +class Interrupt(BaseException): + """Stands for a killed process: not an Exception, so the harness does not catch it.""" + + +def spec(conditions: tuple[Condition, ...] = ALL_CONDITIONS, **changes: Any) -> ExperimentSpec: + settings: dict[str, Any] = { + "name": "p8a-test", + "version": "1", + "protocol_version": "draft", + "objective": "harness check on the mini scenarios", + "hypotheses": ("none: a harness check",), + "conditions": conditions, + } + settings.update(changes) + return ExperimentSpec(**settings) + + +class HarnessCase(unittest.TestCase): + def setUp(self) -> None: + self.dirs = TempDirs() + self.addCleanup(self.dirs.close) + self.mini = mini_root(self.dirs.new()) + self.loaded = load(self.mini.root, manifest_sha256=self.mini.manifest_sha256) + + def experiment(self, conditions: tuple[Condition, ...] = ALL_CONDITIONS, loaded: Any = None): # type: ignore[no-untyped-def] + return plan( + spec(conditions), loaded or self.loaded, environment=ENVIRONMENT, created_at=CREATED + ) + + def run_all( # type: ignore[no-untyped-def] + self, + conditions: tuple[Condition, ...] = ALL_CONDITIONS, + *, + root: Path | None = None, + **options: Any, + ): + experiment = self.experiment(conditions, options.pop("loaded", None)) + out = root or self.dirs.new() + summary = run_experiment(experiment, options.pop("data", self.loaded), out, **options) + return experiment, Layout(out, experiment.experiment_id), summary + + @staticmethod + def decisions(layout: Layout, trial_id: str) -> list[str | None]: + result = read_result(layout, trial_id) + return [ + None if t["decision"] is None else t["decision"]["type"] for t in result["transitions"] + ] + + +class TestDefinition(HarnessCase): + def test_ids_are_deterministic(self) -> None: + one, two = self.experiment(), self.experiment() + self.assertEqual(one.experiment_id, two.experiment_id) + self.assertEqual(one.trial_id(F2, C), two.trial_id(F2, C)) + ids = {one.trial_id(s, c) for s in TARGETS for c in ALL_CONDITIONS} + self.assertEqual(len(ids), len(TARGETS) * len(ALL_CONDITIONS)) + other = plan(spec(version="2"), self.loaded, environment=ENVIRONMENT, created_at=CREATED) + self.assertNotEqual(other.experiment_id, one.experiment_id) + + def test_the_definition_has_the_fields_of_doc_13_and_the_pins(self) -> None: + data = self.experiment().to_dict() + for key in ( + "experiment_id", + "name", + "version", + "objective", + "hypothesis", + "protocol_version", + "fixture_set", + "baseline_set", + "condition_set", + "repetitions", + "seed_policy", + "metrics", + "stopping_policy", + "created_at", + "pins", + ): + self.assertIn(key, data) + pins = data["pins"] + for key in ( + "dail_commit", + "terrapreserve_tag", + "terrapreserve_manifest_sha256", + "invariant_registry", + "policies", + "verifiers", + "python", + "platform", + ): + self.assertIn(key, pins) + self.assertEqual(pins["terrapreserve_manifest_sha256"], self.mini.manifest_sha256) + self.assertEqual( + pins["invariant_registry"], + {"tag": "registry-v1.1", "stored_invariant_registry_version": 2}, + ) + self.assertEqual(data["repetitions"], 1) + self.assertEqual(data["fixture_set"], sorted(TARGETS)) + self.assertEqual(data["condition_set"], ["A", "B", "C", "C_PRIME"]) + + def test_the_definition_cannot_change_once_written(self) -> None: + out = self.dirs.new() + run_experiment(self.experiment((C,)), self.loaded, out) + with self.assertRaises(ExperimentError): + run_experiment(self.experiment((A, C)), self.loaded, out) + + def test_the_lifecycle_of_doc_13(self) -> None: + S = TrialStatus + for path in ( + (S.PLANNED, S.INITIALIZED, S.EXECUTING, S.COMPLETED, S.ANALYZED, S.INCLUDED), + (S.PLANNED, S.INITIALIZED, S.EXECUTING, S.FAILED), + (S.PLANNED, S.INITIALIZED, S.EXECUTING, S.ABORTED), + ): + for before, after in zip(path, path[1:], strict=False): + check_transition(before, after) + for before, after in ( + (S.PLANNED, S.COMPLETED), + (S.COMPLETED, S.EXECUTING), + (S.FAILED, S.COMPLETED), + (S.INITIALIZED, S.COMPLETED), + ): + with self.subTest(before=before, after=after), self.assertRaises(TransitionError): + check_transition(before, after) + + +class TestConditions(unittest.TestCase): + """One run of every condition on every mini target, shared by the tests of this class.""" + + decisions = staticmethod(HarnessCase.decisions) + + @classmethod + def setUpClass(cls) -> None: + cls.dirs = TempDirs() + mini = mini_root(cls.dirs.new()) + loaded = load(mini.root, manifest_sha256=mini.manifest_sha256) + cls.exp = plan(spec(), loaded, environment=ENVIRONMENT, created_at=CREATED) + out = cls.dirs.new() + cls.summary = run_experiment(cls.exp, loaded, out) + cls.layout = Layout(out, cls.exp.experiment_id) + + @classmethod + def tearDownClass(cls) -> None: + cls.dirs.close() + + def test_every_trial_completed(self) -> None: + self.assertEqual(len(self.summary.statuses), len(TARGETS) * 4) + self.assertEqual(set(self.summary.statuses.values()), {TrialStatus.COMPLETED}) + self.assertEqual(self.summary.failures, {}) + + def test_the_safe_repair_is_promoted_by_every_condition(self) -> None: + for condition in ALL_CONDITIONS: + with self.subTest(condition=condition): + tid = self.exp.trial_id(F2, condition) + self.assertEqual(self.decisions(self.layout, tid), ["PROMOTE"]) + + def test_the_regression_is_promoted_only_by_a(self) -> None: + expected = {A: ["PROMOTE"], B: ["REJECT"], C: ["REJECT"], CP: ["REJECT"]} + for condition, decisions in expected.items(): + with self.subTest(condition=condition): + self.assertEqual( + self.decisions(self.layout, self.exp.trial_id(F4, condition)), decisions + ) + + def test_the_change_request_is_promoted_by_every_condition(self) -> None: + for condition in ALL_CONDITIONS: + with self.subTest(condition=condition): + self.assertEqual( + self.decisions(self.layout, self.exp.trial_id(F1, condition)), ["PROMOTE"] + ) + + def test_a_multi_step_repair_uses_the_open_target_on_the_middle_step(self) -> None: + for condition in ALL_CONDITIONS: + with self.subTest(condition=condition): + result = read_result(self.layout, self.exp.trial_id(F8, condition)) + first, last = result["transitions"] + self.assertEqual(first["open_targets"], [SEC1]) + self.assertEqual(last["open_targets"], []) + self.assertEqual(first["decision"]["type"], "PROMOTE") + self.assertEqual(last["decision"]["type"], "PROMOTE") + self.assertEqual(first["new_state"]["invariants"][SEC1]["status"], "VIOLATED") + self.assertEqual(last["new_state"]["invariants"][SEC1]["status"], "PROTECTED") + + def test_a_middle_step_that_does_not_touch_the_target_promotes_everywhere(self) -> None: + # C-210 (closes C-209): DAIL re-checks the VIOLATED target although impact does not list it + for condition in ALL_CONDITIONS: + with self.subTest(condition=condition): + result = read_result(self.layout, self.exp.trial_id(F8_UNTOUCHED, condition)) + first, last = result["transitions"] + self.assertEqual(first["decision"]["type"], "PROMOTE") + self.assertEqual(first["new_state"]["invariants"][SEC1]["status"], "VIOLATED") + self.assertEqual(last["decision"]["type"], "PROMOTE") + self.assertEqual(last["new_state"]["invariants"][SEC1]["status"], "PROTECTED") + + def test_the_not_protected_items_are_recorded(self) -> None: + for condition in (C, CP): + with self.subTest(condition=condition): + first = read_result(self.layout, self.exp.trial_id(F8_UNTOUCHED, condition))[ + "transitions" + ][0] + row = next(w for w in first["work"] if w["target_id"] == "INVARIANT:INV-SEC-001:2") + self.assertEqual(row["origin"], "condition") + self.assertEqual(row["selected_reason"], "NOT_PROTECTED_IN_PARENT") + self.assertEqual(first["work_counts"]["not_protected_in_parent"], 1) + b_first = read_result(self.layout, self.exp.trial_id(F2, B))["transitions"][0] + added = [w for w in b_first["work"] if w["origin"] == "condition"] + self.assertTrue(added) + self.assertEqual({w["selected_reason"] for w in added}, {"CONDITION_SCOPE"}) + + def test_a_rejected_middle_step_truncates_the_rest(self) -> None: + for condition in (B, C, CP): + with self.subTest(condition=condition): + result = read_result(self.layout, self.exp.trial_id(F8_BROKEN, condition)) + first, rest = result["transitions"][0], result["transitions"][1:] + self.assertEqual(first["decision"]["type"], "REJECT") + self.assertEqual([t["status"] for t in rest], ["TRUNCATED"]) + self.assertIsNone(rest[0]["decision"]) + self.assertEqual(rest[0]["step_index"], 2) + + def test_condition_a_holds_only_the_target(self) -> None: + baseline = read_result(self.layout, self.exp.trial_id(F2, A))["baseline"] + self.assertEqual(sorted(baseline["invariants"]), [SEC1]) + change = read_result(self.layout, self.exp.trial_id(F1, A))["baseline"] + self.assertEqual(change["invariants"], {}) + for condition in (B, C, CP): + with self.subTest(condition=condition): + held = read_result(self.layout, self.exp.trial_id(F2, condition))["baseline"] + self.assertEqual( + sorted(held["invariants"]), + ["INV-FUNC-001", "INV-SEC-001", "INV-SEC-002", "INV-SEC-003"], + ) + + def test_baselines_agree_across_conditions(self) -> None: + # the same S0 under B, C and C' (Doc 13 §50 "baseline consistency") + held = [read_result(self.layout, self.exp.trial_id(F2, c))["baseline"] for c in (B, C, CP)] + # state_hash covers the lineage (C-33), and every trial has its own: compare the resources + self.assertEqual(held[0]["resource_set_hash"], held[1]["resource_set_hash"]) + self.assertEqual(held[1]["resource_set_hash"], held[2]["resource_set_hash"]) + self.assertNotEqual(held[0]["state_hash"], held[1]["state_hash"]) + self.assertEqual(held[0]["invariants"], held[1]["invariants"]) + self.assertEqual(held[1]["invariants"], held[2]["invariants"]) + self.assertEqual(held[0]["invariants"][SEC1]["status"], "VIOLATED") + + def test_work_differs_by_condition(self) -> None: + def counts(condition: Condition) -> dict[str, int]: + first = read_result(self.layout, self.exp.trial_id(F2, condition))["transitions"][0] + return first["work_counts"] + + self.assertEqual(counts(A)["gate"], 2) + self.assertEqual(counts(A)["invariant"], 1) + self.assertEqual(counts(B)["invariant"], 4) + self.assertLessEqual(counts(C)["invariant"], 4) + a_change = read_result(self.layout, self.exp.trial_id(F1, A))["transitions"][0] + self.assertEqual(a_change["work_counts"]["invariant"], 0) + self.assertEqual(a_change["work_counts"]["baseline_applicable"], 0) + self.assertEqual(a_change["work_counts"]["baseline_not_applicable"], 0) + + def test_c_prime_uses_address_only_identity(self) -> None: + cp = read_result(self.layout, self.exp.trial_id(F2, CP))["transitions"][0] + c = read_result(self.layout, self.exp.trial_id(F2, C))["transitions"][0] + self.assertEqual(cp["identity"]["algorithm_version"], "dail-identity-addr-1") + self.assertEqual(c["identity"]["algorithm_version"], "dail-identity-1") + + def test_a_transition_records_what_section_4_3_lists(self) -> None: + result = read_result(self.layout, self.exp.trial_id(F8, C)) + for transition in result["transitions"]: + self.assertEqual(set(transition), TRANSITION_FIELDS) + self.assertEqual(transition["correlation_id"], result["correlation_id"]) + self.assertEqual( + transition["registry"], + {"stored_invariant_registry_version": 2, "means": "registry-v1.1"}, + ) + self.assertEqual(transition["outcome_class"], "DAIL_OUTCOME") + self.assertTrue(transition["admission"]["content_hash_matches_manifest"]) + for item in transition["required"]: + self.assertIn(item["source"], ("VERIFIED", "CARRIED_FORWARD", None)) + for work in transition["work"]: + self.assertIn("applicable", work) + self.assertIn(work["origin"], ("impact", "condition", "gate")) + for pair in transition["identity"]["pairs"]: + self.assertEqual( + set(pair), {"trusted_ref", "candidate_ref", "change_type", "moved"} + ) + self.assertEqual(result["status"], "COMPLETED") + self.assertIsNone(result["failure"]) + + def test_the_layout_of_section_41(self) -> None: + root = self.layout.experiment_dir + self.assertEqual(root.parent.name, "experiment") + for name in ("definition", "fixtures", "trials"): + self.assertTrue((root / name).is_dir(), name) + attempt = self.layout.attempt_dir(self.exp.trial_id(F2, C), 1) + for name in ("baseline", "generation", "analysis", "verification", "decision", "metrics"): + self.assertTrue((attempt / name).is_dir(), name) + definition = json.loads((root / "definition" / "experiment.json").read_text("utf-8")) + self.assertEqual(definition["experiment_id"], self.exp.experiment_id) + + def test_the_lifecycle_events_of_a_completed_trial(self) -> None: + events = read_events(self.layout, self.exp.trial_id(F2, C)) + self.assertEqual( + [e["status"] for e in events], ["PLANNED", "INITIALIZED", "EXECUTING", "COMPLETED"] + ) + + +class TestFailures(HarnessCase): + def test_a_harness_failure_marks_the_trial_failed_and_is_no_decision(self) -> None: + experiment = self.experiment((C,)) + broken = experiment.trial_id(F2, C) + + def hook(trial_id: str, stage: str) -> None: + if trial_id == broken and stage == "step-1": + raise RuntimeError("injected harness fault") + + out = self.dirs.new() + summary = run_experiment(experiment, self.loaded, out, hook=hook) + layout = Layout(out, experiment.experiment_id) + self.assertIs(summary.statuses[broken], TrialStatus.FAILED) + self.assertIn("injected harness fault", summary.failures[broken]) + result = read_result(layout, broken) + self.assertEqual(result["status"], "FAILED") + self.assertEqual(result["failure"]["class"], "HARNESS_FAILURE") + self.assertEqual(result["transitions"], []) + others = {t for t in summary.statuses if t != broken} + self.assertEqual({summary.statuses[t] for t in others}, {TrialStatus.COMPLETED}) + + def test_an_input_failure_stops_the_run_before_any_trial(self) -> None: + main = self.mini.root / "scenarios" / "dev" / F2 / "step_1" / "main.tf" + main.write_bytes(main.read_bytes() + b"# tampered\n") + from experiments.loader import LoaderError + + with self.assertRaises(LoaderError): + load(self.mini.root, manifest_sha256=self.mini.manifest_sha256) + + def test_a_step_changed_after_loading_is_an_input_failure(self) -> None: + experiment = self.experiment((C,)) + main = self.mini.root / "scenarios" / "dev" / F2 / "step_1" / "main.tf" + main.write_bytes(main.read_bytes() + b"# tampered\n") + out = self.dirs.new() + summary = run_experiment(experiment, self.loaded, out) + tid = experiment.trial_id(F2, C) + self.assertIs(summary.statuses[tid], TrialStatus.FAILED) + result = read_result(Layout(out, experiment.experiment_id), tid) + self.assertEqual(result["failure"]["class"], "INPUT_FAILURE") + self.assertEqual(result["transitions"], []) + + +class TestRerun(HarnessCase): + def snapshot(self, root: Path) -> dict[Path, bytes]: + return {p: p.read_bytes() for p in root.rglob("*") if p.is_file() and p.suffix != ".sqlite"} + + def test_an_interrupted_run_resumes_without_overwriting(self) -> None: + experiment = self.experiment((A, C)) + victim = experiment.trial_id(F8, C) + + def hook(trial_id: str, stage: str) -> None: + if trial_id == victim and stage == "step-2": + raise Interrupt() + + out = self.dirs.new() + with self.assertRaises(Interrupt): + run_experiment(experiment, self.loaded, out, hook=hook) + layout = Layout(out, experiment.experiment_id) + self.assertEqual(read_events(layout, victim)[-1]["status"], "EXECUTING") + before = self.snapshot(out) + summary = run_experiment(experiment, self.loaded, out) + self.assertEqual(set(summary.statuses.values()), {TrialStatus.COMPLETED}) + self.assertIn(victim, summary.attempted) + for path, data in before.items(): + with self.subTest(path=str(path.relative_to(out))): + now = path.read_bytes() + if path.name == "events.jsonl": + self.assertTrue(now.startswith(data)) # append-only + else: + self.assertEqual(now, data) + statuses = [(e["attempt"], e["status"]) for e in read_events(layout, victim)] + self.assertIn((1, "ABORTED"), statuses) + self.assertEqual(statuses[-1], (2, "COMPLETED")) + self.assertEqual(read_result(layout, victim)["attempt"], 2) + completed_before = [ + t + for t in summary.statuses + if t != victim and (layout.trial_dir(t) / "attempts" / "1" / "result.json") in before + ] + self.assertTrue(completed_before) + for tid in completed_before: + self.assertIn(tid, summary.skipped) + + def test_an_initialized_but_unstarted_trial_is_rerun(self) -> None: + experiment = self.experiment((C,)) + victim = experiment.trial_id(F1, C) + + def hook(trial_id: str, stage: str) -> None: + if trial_id == victim and stage == "initialized": + raise Interrupt() + + out = self.dirs.new() + with self.assertRaises(Interrupt): + run_experiment(experiment, self.loaded, out, hook=hook) + summary = run_experiment(experiment, self.loaded, out) + self.assertIs(summary.statuses[victim], TrialStatus.COMPLETED) + events = read_events(Layout(out, experiment.experiment_id), victim) + self.assertIn((1, "ABORTED"), [(e["attempt"], e["status"]) for e in events]) + + def test_a_completed_run_is_skipped_entirely(self) -> None: + experiment, layout, _ = self.run_all((C,)) + before = self.snapshot(layout.root) + summary = run_experiment(experiment, self.loaded, layout.root) + self.assertEqual(summary.attempted, ()) + self.assertEqual(len(summary.skipped), len(TARGETS)) + self.assertEqual(self.snapshot(layout.root), before) + + +def records(layout: Layout) -> dict[str, bytes]: + """Every trial record of a run, keyed by its path inside the experiment directory, without the + wall-clock files (timing and lifecycle events) and the database files.""" + root = layout.experiment_dir / "trials" + return { + p.relative_to(root).as_posix(): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() + and p.name not in ("timing.json", "events.jsonl") + and not p.name.startswith("dail.sqlite") + } + + +class TestReproducibility(HarnessCase): + def test_running_a_trial_twice_gives_identical_records(self) -> None: + _, first, _ = self.run_all((C, A)) + _, second, _ = self.run_all((C, A)) + self.assertEqual(records(first), records(second)) + self.assertGreater(len(records(first)), 0) + + def test_corrupted_labels_change_no_trial_record(self) -> None: + corrupted_root = mini_root(self.dirs.new(), corrupt=True) + corrupted = load(corrupted_root.root, manifest_sha256=corrupted_root.manifest_sha256) + _, clean, _ = self.run_all() + _, dirty, _ = self.run_all(loaded=corrupted, data=corrupted) + self.assertEqual(records(clean), records(dirty)) + + def test_concurrent_trials_are_isolated(self) -> None: + _, sequential, _ = self.run_all((A, C)) + _, parallel, summary = self.run_all((A, C), workers=4) + self.assertEqual(set(summary.statuses.values()), {TrialStatus.COMPLETED}) + self.assertEqual(records(sequential), records(parallel)) + + def test_each_trial_has_its_own_store_and_lineage(self) -> None: + experiment, layout, _ = self.run_all((A, C)) + lineages, stores = set(), set() + for tid in (experiment.trial_id(s, c) for s in TARGETS for c in (A, C)): + result = read_result(layout, tid) + lineages.add(result["lineage_id"]) + stores.add(layout.attempt_dir(tid, 1) / "store" / "dail.sqlite") + self.assertEqual(len(lineages), len(TARGETS) * 2) + self.assertTrue(all(store.is_file() for store in stores)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_experiment_loader.py b/tests/unit/test_experiment_loader.py new file mode 100644 index 0000000..12263cc --- /dev/null +++ b/tests/unit/test_experiment_loader.py @@ -0,0 +1,273 @@ +"""The TerraPreserve loader of the experiment harness (P8a; C-201, C-204; work package §3, §4.2). + +Expected values come from the quoted text: + +"steps[k].content_hash = SHA-256 of the newline-joined, path-sorted lines ':' of the step directory; scenarios/V1_MANIFEST.json lists every +dev scenario's scenario_json_sha256 and step hashes, and its SHA-256 at tag terrapreserve-v1.0 is +665b326a... (pin this value). Trials run only scenarios with v1_role = 'target'." +§4.2: "Verifies the manifest pin, every scenario_json_sha256, and every step's content_hash ... Any +mismatch stops the run before any trial starts. Gives execution only: scenario_id, family, split, +template_id or lineage id, the step directories and hashes, target.kind, target.invariant_id, +protected_invariants, endpoints ... and the primary variable_values." +"The loader refuses split = 'test' (a test enforces it)." +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import unittest +from pathlib import Path + +from experiments.loader import ( + MANIFEST_SHA256, + TERRAPRESERVE_TAG, + LoaderError, + ScenarioInput, + StepInput, + load, + step_content_hash, +) +from tests.experiment_builders import ( + F1, + F8, + RESERVE, + TARGETS, + TempDirs, + content_hash, + mini_root, +) + +QUOTED_PIN = "665b326a024d295f727cd5648ffc7e0d375b9300584871f238c6bfa48edd2e2d" +EXECUTION_FIELDS = { + "scenario_id", + "family", + "split", + "template_id", + "source_lineage_id", + "target_kind", + "target_invariant_id", + "protected_invariants", + "source_endpoint", + "destination_endpoint", + "variable_values", + "steps", +} + + +class LoaderCase(unittest.TestCase): + def setUp(self) -> None: + self.dirs = TempDirs() + self.addCleanup(self.dirs.close) + self.mini = mini_root(self.dirs.new()) + + def load(self, **options: object): # type: ignore[no-untyped-def] + options.setdefault("manifest_sha256", self.mini.manifest_sha256) + return load(self.mini.root, **options) # type: ignore[arg-type] + + +class TestPins(unittest.TestCase): + def test_the_manifest_pin_is_the_quoted_value(self) -> None: + self.assertEqual(MANIFEST_SHA256, QUOTED_PIN) + self.assertEqual(TERRAPRESERVE_TAG, "terrapreserve-v1.0") + + +class TestContentHash(unittest.TestCase): + def test_it_follows_the_quoted_definition(self) -> None: + dirs = TempDirs() + self.addCleanup(dirs.close) + step = dirs.new() + (step / "main.tf").write_bytes(b"a\n") + (step / "sub").mkdir() + (step / "sub" / "x.tf").write_bytes(b"x") + lines = [ + f"main.tf:{hashlib.sha256(b'a' + bytes([10])).hexdigest()}", + f"sub/x.tf:{hashlib.sha256(b'x').hexdigest()}", + ] + expected = hashlib.sha256("\n".join(lines).encode("utf-8")).hexdigest() + self.assertEqual(step_content_hash(step), expected) + + def test_the_lines_are_sorted_by_path(self) -> None: + # "a.tf" sorts before "a.tf.json" as a path; as whole lines ':' would sort after '.' + dirs = TempDirs() + self.addCleanup(dirs.close) + step = dirs.new() + (step / "a.tf").write_bytes(b"1") + (step / "a.tf.json").write_bytes(b"2") + lines = [ + f"a.tf:{hashlib.sha256(b'1').hexdigest()}", + f"a.tf.json:{hashlib.sha256(b'2').hexdigest()}", + ] + expected = hashlib.sha256("\n".join(lines).encode("utf-8")).hexdigest() + self.assertEqual(step_content_hash(step), expected) + self.assertEqual(content_hash(step), expected) # the builder's own implementation agrees + + +class TestLoading(LoaderCase): + def test_it_loads_the_targets_only(self) -> None: + loaded = self.load() + self.assertEqual(tuple(s.scenario_id for s in loaded.scenarios), TARGETS) + self.assertNotIn(RESERVE, {s.scenario_id for s in loaded.scenarios}) + self.assertEqual(loaded.manifest_sha256, self.mini.manifest_sha256) + + def test_it_gives_execution_only_the_listed_fields(self) -> None: + self.assertEqual({f.name for f in dataclasses.fields(ScenarioInput)}, EXECUTION_FIELDS) + self.assertEqual( + {f.name for f in dataclasses.fields(StepInput)}, {"index", "directory", "content_hash"} + ) + + def test_the_fields_are_those_of_the_scenario(self) -> None: + loaded = {s.scenario_id: s for s in self.load().scenarios} + f8 = loaded[F8] + self.assertEqual(f8.family, "F8") + self.assertEqual(f8.split, "dev") + self.assertEqual(f8.template_id, "tpl-mini") + self.assertIsNone(f8.source_lineage_id) + self.assertEqual((f8.target_kind, f8.target_invariant_id), ("repair", "INV-SEC-001")) + self.assertEqual( + (f8.source_endpoint, f8.destination_endpoint), + ("aws_instance.app", "aws_db_instance.database"), + ) + self.assertEqual( + f8.protected_invariants, + ( + ("INV-FUNC-001", "1.0"), + ("INV-SEC-001", "1.0"), + ("INV-SEC-002", "1.0"), + ("INV-SEC-003", "1.0"), + ), + ) + self.assertEqual(dict(f8.variable_values), {}) + self.assertEqual([s.index for s in f8.steps], [0, 1, 2]) + base = self.mini.root / "scenarios" / "dev" / F8 + for step in f8.steps: + self.assertEqual(step.directory, base / f"step_{step.index}") + self.assertEqual(step.content_hash, content_hash(step.directory)) + f1 = loaded[F1] + self.assertEqual((f1.target_kind, f1.target_invariant_id), ("change_request", None)) + + def test_the_fixture_record_names_every_hash(self) -> None: + loaded = self.load() + fixtures = {f.scenario_id: f for f in loaded.fixtures} + self.assertEqual(set(fixtures), set(TARGETS)) + raw = (self.mini.root / "scenarios" / "dev" / F8 / "scenario.json").read_bytes() + self.assertEqual(fixtures[F8].scenario_json_sha256, hashlib.sha256(raw).hexdigest()) + self.assertEqual(len(fixtures[F8].step_hashes), 3) + + def test_the_test_list_of_the_manifest_is_never_read(self) -> None: + # its entry points at a directory that does not exist; loading the dev split succeeds + self.assertFalse((self.mini.root / "scenarios" / "test").exists()) + self.assertEqual(len(self.load().scenarios), len(TARGETS)) + + +class TestRefusals(LoaderCase): + def test_the_test_split_is_refused(self) -> None: + with self.assertRaisesRegex(LoaderError, "test split"): + self.load(split="test") + + def test_any_other_split_is_refused(self) -> None: + for split in ("TEST", "dev ", "private", ""): + with self.subTest(split=split), self.assertRaises(LoaderError): + self.load(split=split) + + def test_a_wrong_manifest_pin_is_refused(self) -> None: + with self.assertRaisesRegex(LoaderError, "manifest"): + self.load(manifest_sha256="0" * 64) + with self.assertRaisesRegex(LoaderError, "manifest"): + load(self.mini.root) # the default pin is the real TerraPreserve one + + def test_a_changed_step_file_is_refused(self) -> None: + main = self.mini.root / "scenarios" / "dev" / F8 / "step_1" / "main.tf" + main.write_bytes(main.read_bytes() + b"\n# changed\n") + with self.assertRaisesRegex(LoaderError, F8): + self.load() + + def test_an_added_step_file_is_refused(self) -> None: + extra = self.mini.root / "scenarios" / "dev" / F1 / "step_0" / "extra.tf" + extra.write_bytes(b"# extra\n") + with self.assertRaisesRegex(LoaderError, F1): + self.load() + + def test_a_changed_scenario_file_is_refused(self) -> None: + path = self.mini.root / "scenarios" / "dev" / F8 / "scenario.json" + path.write_bytes(path.read_bytes() + b" ") + with self.assertRaisesRegex(LoaderError, F8): + self.load() + + def test_a_missing_step_directory_is_refused(self) -> None: + step = self.mini.root / "scenarios" / "dev" / F8 / "step_2" + for path in step.iterdir(): + path.unlink() + step.rmdir() + with self.assertRaisesRegex(LoaderError, F8): + self.load() + + def test_a_scenario_whose_step_hash_disagrees_with_the_manifest_is_refused(self) -> None: + def edit(scenario: dict[str, object]) -> None: + if scenario["scenario_id"] == F8: + steps = scenario["steps"] + assert isinstance(steps, list) + steps[1] = {**steps[1], "content_hash": "f" * 64} + + dirs = TempDirs() + self.addCleanup(dirs.close) + mini = mini_root(dirs.new(), edit=edit) + with self.assertRaisesRegex(LoaderError, F8): + load(mini.root, manifest_sha256=mini.manifest_sha256) + + def test_a_scenario_that_says_it_is_not_dev_is_refused(self) -> None: + def edit(scenario: dict[str, object]) -> None: + if scenario["scenario_id"] == F1: + scenario["split"] = "test" + + dirs = TempDirs() + self.addCleanup(dirs.close) + mini = mini_root(dirs.new(), edit=edit) + with self.assertRaisesRegex(LoaderError, F1): + load(mini.root, manifest_sha256=mini.manifest_sha256) + + def test_a_real_world_scenario_names_its_source_lineage(self) -> None: + # "template_id or lineage id": a real-world scenario has no template + def edit(scenario: dict[str, object]) -> None: + if scenario["scenario_id"] == F1: + del scenario["template_id"] + scenario["source"] = {"repo_lineage_id": "lin-mini-1", "tier": "real_world"} + + dirs = TempDirs() + self.addCleanup(dirs.close) + mini = mini_root(dirs.new(), edit=edit) + loaded = { + s.scenario_id: s + for s in load(mini.root, manifest_sha256=mini.manifest_sha256).scenarios + } + self.assertIsNone(loaded[F1].template_id) + self.assertEqual(loaded[F1].source_lineage_id, "lin-mini-1") + + def test_a_scenario_with_neither_template_nor_lineage_is_refused(self) -> None: + def edit(scenario: dict[str, object]) -> None: + if scenario["scenario_id"] == F1: + del scenario["template_id"] + + dirs = TempDirs() + self.addCleanup(dirs.close) + mini = mini_root(dirs.new(), edit=edit) + with self.assertRaisesRegex(LoaderError, F1): + load(mini.root, manifest_sha256=mini.manifest_sha256) + + def test_a_missing_root_is_refused(self) -> None: + with self.assertRaises(LoaderError): + load(Path(self.mini.root) / "nowhere", manifest_sha256=self.mini.manifest_sha256) + + def test_nothing_partial_is_returned(self) -> None: + # one bad scenario stops the whole load: no list of the good ones comes back + path = self.mini.root / "scenarios" / "dev" / F1 / "scenario.json" + data = json.loads(path.read_text(encoding="utf-8")) + path.write_text(json.dumps(data), encoding="utf-8") # same content, other bytes + with self.assertRaises(LoaderError): + self.load() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_experiment_terrapreserve.py b/tests/unit/test_experiment_terrapreserve.py new file mode 100644 index 0000000..4d85aa3 --- /dev/null +++ b/tests/unit/test_experiment_terrapreserve.py @@ -0,0 +1,63 @@ +"""The loader and one trial on the real TerraPreserve v1.0 dev split (P8a; work package §3, Step 2). + +Runs only when ``TERRAPRESERVE_ROOT`` names a TerraPreserve checkout at tag terrapreserve-v1.0 +(skipped in CI, which has no TerraPreserve). Read only: nothing under the root is written. Never the +test split. + +Expected values from the quoted text: "its SHA-256 at tag terrapreserve-v1.0 is 665b326a..."; +"Trials run only scenarios with v1_role = 'target' (224 in dev)". +""" + +from __future__ import annotations + +import os +import unittest +from datetime import UTC, datetime +from pathlib import Path + +from experiments.definition import Condition, ExperimentSpec, TrialStatus +from experiments.loader import load +from experiments.runner import plan, run_experiment +from tests.experiment_builders import TempDirs + +ROOT = os.environ.get("TERRAPRESERVE_ROOT") + + +@unittest.skipUnless(ROOT, "set TERRAPRESERVE_ROOT to a TerraPreserve v1.0 checkout") +class TestRealTerraPreserve(unittest.TestCase): + def test_the_dev_targets_load_under_the_pin(self) -> None: + assert ROOT is not None + loaded = load(Path(ROOT)) + self.assertEqual(len(loaded.scenarios), 224) + self.assertEqual({s.split for s in loaded.scenarios}, {"dev"}) + self.assertEqual( + loaded.manifest_sha256, + "665b326a024d295f727cd5648ffc7e0d375b9300584871f238c6bfa48edd2e2d", + ) + + def test_one_multi_step_trial_completes(self) -> None: + assert ROOT is not None + loaded = load(Path(ROOT)) + first = next(s for s in loaded.scenarios if s.family == "F8") + dirs = TempDirs() + self.addCleanup(dirs.close) + experiment = plan( + ExperimentSpec( + name="p8a-real-data-test", + version="1", + protocol_version="draft", + objective="loader and harness check", + hypotheses=("none",), + conditions=(Condition.C,), + ), + loaded, + environment={"dail_commit": "test", "terrapreserve_tag": "terrapreserve-v1.0"}, + created_at=datetime(2026, 11, 12, tzinfo=UTC), + only=(first.scenario_id,), + ) + summary = run_experiment(experiment, loaded, dirs.new()) + self.assertEqual(set(summary.statuses.values()), {TrialStatus.COMPLETED}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_identity_strategy.py b/tests/unit/test_identity_strategy.py index 58c410b..8e94603 100644 --- a/tests/unit/test_identity_strategy.py +++ b/tests/unit/test_identity_strategy.py @@ -102,19 +102,55 @@ def test_no_production_module_names_the_ablation(self) -> None: self.assertNotIn("ADDRESS_ONLY", path.read_text(encoding="utf-8")) def test_no_configuration_key_or_environment_variable_selects_a_strategy(self) -> None: - """No file where a switch could live mentions one: the configuration data and schema, the - configuration code, and the scripts.""" + """No file where a switch could live mentions one: the configuration data and schema, and + the scripts. Amended in P8a (C-208, the user's decision of 12 Nov 2026): the orchestrator + in core/application now has the injection point the P8a work package asks for + (``identity_strategy``, default ``EvidenceIdentity``), so core/application is checked by + ``test_the_application_layer_reads_no_strategy_switch`` below instead of by its words.""" words = re.compile(r"identity[_-]?strategy|ablation|address[_-]?only|DAIL_IDENTITY", re.I) places = [ *(path for path in (REPO_ROOT / "config").rglob("*") if path.is_file()), *self.python_files("scripts"), - *(REPO_ROOT / "core" / "application").glob("*.py"), + REPO_ROOT / "core" / "application" / "config.py", ] self.assertTrue(places) for path in places: with self.subTest(path=str(path.relative_to(REPO_ROOT))): self.assertIsNone(words.search(path.read_text(encoding="utf-8"))) + def test_the_application_layer_reads_no_strategy_switch(self) -> None: + """C-208: in core/application no strategy comes from configuration or the environment. + No module reads the environment at all (``os.environ``, ``getenv``), none names the + ablation, no ``get``/``[]`` lookup uses a key that names a strategy, and the orchestrator's + default is the full strategy.""" + import ast + import inspect + + from core.application.orchestrator import EvaluationOrchestrator + from core.identity.strategy import DEFAULT_STRATEGY, EvidenceIdentity + + words = re.compile(r"identity|strategy|ablation|address[_-]?only", re.I) + for path in sorted((REPO_ROOT / "core" / "application").glob("*.py")): + text = path.read_text(encoding="utf-8") + with self.subTest(path=str(path.relative_to(REPO_ROOT))): + self.assertNotIn("AddressOnlyIdentity", text) + self.assertNotIn("ADDRESS_ONLY", text) + for node in ast.walk(ast.parse(text)): + if path.name != "config.py": # the loader reads the CSX_ variables only + name = getattr(node, "attr", None) or getattr(node, "id", None) + self.assertNotIn(name, ("environ", "getenv")) + keys: list[ast.expr] = [] + if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "get": + keys = list(node.args[:1]) + elif isinstance(node, ast.Subscript): + keys = [node.slice] + for key in keys: + if isinstance(key, ast.Constant) and isinstance(key.value, str): + self.assertIsNone(words.search(key.value), key.value) + default = inspect.signature(EvaluationOrchestrator.__init__).parameters["identity_strategy"] + self.assertIs(default.default, DEFAULT_STRATEGY) + self.assertIsInstance(DEFAULT_STRATEGY, EvidenceIdentity) + def test_the_identity_package_reads_no_environment(self) -> None: for path in (REPO_ROOT / "core" / "identity").glob("*.py"): text = path.read_text(encoding="utf-8") diff --git a/tests/unit/test_impact_domain.py b/tests/unit/test_impact_domain.py index 2d2add0..b544998 100644 --- a/tests/unit/test_impact_domain.py +++ b/tests/unit/test_impact_domain.py @@ -18,6 +18,7 @@ from core.domain.impact import ( AFFECTED_CODES, IMPACT_ALGORITHM_VERSION, + POLICY_CODES, REASON_DEFINITIONS, UNAFFECTED_CODES, UNCERTAIN_CODES, @@ -75,12 +76,24 @@ def test_the_reason_codes_are_the_closed_list_of_c_138(self) -> None: "NO_PRIOR_EVIDENCE", "INVARIANT_VERSION_CHANGED", "VERIFIER_CHANGED", + # P8a-2 (C-211): the reasons of work that a scope policy adds, never impact's + "CONDITION_SCOPE", + "NOT_PROTECTED_IN_PARENT", }, ) self.assertEqual(set(REASON_DEFINITIONS), set(ReasonCode)) + self.assertEqual( + POLICY_CODES, {ReasonCode.CONDITION_SCOPE, ReasonCode.NOT_PROTECTED_IN_PARENT} + ) def test_every_code_is_in_exactly_one_group(self) -> None: - groups = [AFFECTED_CODES, UNCERTAIN_CODES, UNAFFECTED_CODES, {ReasonCode.CREATED_BASELINE}] + groups = [ + AFFECTED_CODES, + UNCERTAIN_CODES, + UNAFFECTED_CODES, + {ReasonCode.CREATED_BASELINE}, + POLICY_CODES, + ] for code in ReasonCode: self.assertEqual(sum(code in g for g in groups), 1, code) @@ -623,3 +636,36 @@ def test_a_float_version_is_refused_everywhere(self) -> None: if __name__ == "__main__": unittest.main() + + +class TestPolicyReasonCodes(unittest.TestCase): + """C-211: CONDITION_SCOPE and NOT_PROTECTED_IN_PARENT are reasons of invariant work that a + scope policy adds; a resource item cannot carry them and an impact report never does.""" + + def item(self, reason: ReasonCode, target: str = "INVARIANT:INV-SEC-002:1"): # type: ignore[no-untyped-def] + from core.domain.impact import VerificationWorkItem, WorkPriority, WorkTargetType + + kind = ( + WorkTargetType.INVARIANT if target.startswith("INVARIANT:") else WorkTargetType.RESOURCE + ) + return VerificationWorkItem( + target, kind, reason, ("aws_security_group",), "L1", WorkPriority.BLOCKING + ) + + def test_an_invariant_item_may_carry_them(self) -> None: + for code in POLICY_CODES: + with self.subTest(code=code): + self.assertIs(self.item(code).reason, code) + + def test_a_resource_item_may_not(self) -> None: + for code in POLICY_CODES: + with self.subTest(code=code), self.assertRaises(DomainValidationError): + self.item(code, "RESOURCE:aws_subnet.new") + + def test_an_impact_report_never_carries_them(self) -> None: + for code in POLICY_CODES: + with self.subTest(code=code): + sec = affected_invariant() + bad = dataclasses.replace(work_for(sec), reason=code) + with self.assertRaises(DomainValidationError): + report(affected=[sec], work=[bad, baseline_work("aws_internet_gateway.new")]) diff --git a/tests/unit/test_llm_patch_validator.py b/tests/unit/test_llm_patch_validator.py index b944d03..f6d4850 100644 --- a/tests/unit/test_llm_patch_validator.py +++ b/tests/unit/test_llm_patch_validator.py @@ -335,6 +335,55 @@ def test_a_secret_argument_from_a_variable_is_allowed_only_when_declared(self) - # var.db_password is not declared in the candidate: reference check (INVALID_OUTPUT) self.assert_fails(self.run_patch(patch, [SG_APP]), F.INVALID_OUTPUT) + def test_a_secret_argument_from_a_local_value_is_a_policy_violation(self) -> None: + # P7b verdict limit 2 (P8a-2): only a var. reference may carry a new credential; a local + # value that the parent declares is still a POLICY_VIOLATION + import shutil + import tempfile + from pathlib import Path + + from core.terraform_model.normalization import normalize_directory + from llm.validation.patch import PatchValidator + from tests.identity_builders import NOW + from tests.llm_patch_builders import BASELINE, scope, trusted + + with tempfile.TemporaryDirectory(prefix="dail-p8a2-") as tmp: + parent = Path(tmp) / "parent" + shutil.copytree(BASELINE, parent, ignore=shutil.ignore_patterns("expected.json")) + locals_text = "locals {" + chr(10) + ' x = "from-a-local"' + chr(10) + "}" + chr(10) + (parent / "locals.tf").write_text(locals_text, "utf-8") + outcome = normalize_directory(parent, None, now=NOW) + self.assertEqual(outcome.resource_set_hash, trusted().resource_set_hash()) + db = block( + "aws_db_instance", + "database", + 'identifier = "dail-canonical-db"', + 'engine = "postgres"', + 'instance_class = "db.t3.micro"', + "allocated_storage = 20", + 'username = "dbadmin"', + "password = local.x", + "db_subnet_group_name = aws_db_subnet_group.db.name", + "vpc_security_group_ids = [aws_security_group.db.id]", + "publicly_accessible = false", + "skip_final_snapshot = true", + ) + parsed = parse_response(repair(db, ["aws_db_instance.database"])) + assert parsed.response is not None + work = Path(tmp) / "work" + work.mkdir() + checked = PatchValidator( + parent_dir=parent, + trusted=trusted(), + scope=scope(), + work_root=work, + variables=None, + now=NOW, + ).validate(parsed.response, request_id=REQUEST_ID, patch_id=PATCH_ID) + self.assertIs(checked.failure_class, F.POLICY_VIOLATION, checked.errors) + self.assertIsNone(checked.build) + self.assertEqual(list(work.iterdir()), []) + def test_an_aws_access_key_id_anywhere_is_a_policy_violation(self) -> None: key = ( "AK" diff --git a/tests/unit/test_p8a2_dail_scope.py b/tests/unit/test_p8a2_dail_scope.py new file mode 100644 index 0000000..50c8f4c --- /dev/null +++ b/tests/unit/test_p8a2_dail_scope.py @@ -0,0 +1,177 @@ +"""DAIL re-checks every invariant that is not PROTECTED in the parent (P8a-2; C-210, closes C-209; +C-211 reason codes). + +Expected values come from the decided text (C-210): "DAIL's verification work for a candidate is the +impact work set plus one invariant item for every invariant whose reference in the parent trusted +state is not PROTECTED (VIOLATED or UNCERTAIN) and that impact did not already list ... It applies +to conditions C and C' (both use DailScopePolicy). These items count as verification work. A +VIOLATED reference is never carried forward (C-159 unchanged)." C-211: condition-added items carry +CONDITION_SCOPE, the U_t items NOT_PROTECTED_IN_PARENT; none carries the +SCOPE_TYPE_CHANGED stand-in. + +The canonical baseline holds INV-SEC-001 VIOLATED (public SSH) and the other three PROTECTED. The +"resize" step changes only the application instance's type: it does not touch INV-SEC-001's +security-group footprint. +""" + +from __future__ import annotations + +import shutil +import tempfile +import unittest +from pathlib import Path + +from core.application.orchestrator import ResultKind +from core.application.scope_policy import ( + DailScopePolicy, + StatefulFullScopePolicy, + StatelessScopePolicy, +) +from core.domain.enums import InvariantStatus +from core.domain.impact import ReasonCode, WorkPriority, WorkTargetType +from core.identity.strategy import AddressOnlyIdentity +from tests import impact_builders as ib +from tests.promotion_builders import trusted_state +from tests.promotion_orchestrator_builders import ( + BASELINE_DIR, + PATCH_SAFE_DIR, + OrchestratorCase, +) + +SEC1, SEC2, SEC3, FUNC1 = "INV-SEC-001", "INV-SEC-002", "INV-SEC-003", "INV-FUNC-001" +NOT_PROTECTED = ReasonCode.NOT_PROTECTED_IN_PARENT +CONDITION = ReasonCode.CONDITION_SCOPE + + +def resized(root: Path, source: Path, name: str) -> Path: + """``source`` with aws_instance.app resized (nothing else changes).""" + target = root / name + shutil.copytree(source, target, ignore=shutil.ignore_patterns("expected.json")) + main = target / "main.tf" + text = main.read_text(encoding="utf-8") + old = 'instance_type = "t3.micro"' + assert text.count(old) == 1 + main.write_text(text.replace(old, 'instance_type = "t3.small"'), "utf-8") + return target + + +class TestDailScopePolicySelect(unittest.TestCase): + def setUp(self) -> None: + # impact lists INV-SEC-001 (affected) and INV-FUNC-001 (uncertain) and one baseline item + self.report = ib.report() + self.protected = tuple(ib.registry().values()) + + def refs(self, **statuses: InvariantStatus): # type: ignore[no-untyped-def] + wanted = {SEC1: InvariantStatus.PROTECTED} + wanted.update({k.replace("_", "-"): v for k, v in statuses.items()}) + return trusted_state(wanted).invariant_refs + + def test_it_has_a_new_version(self) -> None: + self.assertEqual(DailScopePolicy.version, "dail-scope-2") + + def test_a_parent_with_every_invariant_protected_changes_nothing(self) -> None: + selected = DailScopePolicy().select(self.report, self.protected, parent_refs=self.refs()) + self.assertEqual(selected, self.report.verification_work) + + def test_without_the_parent_it_is_the_impact_work_set(self) -> None: + self.assertEqual( + DailScopePolicy().select(self.report, self.protected), self.report.verification_work + ) + + def test_a_violated_invariant_impact_did_not_list_is_added(self) -> None: + refs = self.refs(INV_SEC_002=InvariantStatus.VIOLATED) + selected = DailScopePolicy().select(self.report, self.protected, parent_refs=refs) + added = [w for w in selected if w not in self.report.verification_work] + self.assertEqual([w.target_id for w in added], ["INVARIANT:INV-SEC-002:1"]) + self.assertIs(added[0].reason, NOT_PROTECTED) + self.assertIs(added[0].priority, WorkPriority.BLOCKING) + self.assertIs(added[0].target_type, WorkTargetType.INVARIANT) + for item in self.report.verification_work: + self.assertIn(item, selected) + self.assertEqual(list(selected), sorted(selected, key=lambda w: w.sort_key())) + + def test_an_uncertain_invariant_is_added_too(self) -> None: + refs = self.refs(INV_SEC_003=InvariantStatus.UNCERTAIN) + selected = DailScopePolicy().select(self.report, self.protected, parent_refs=refs) + self.assertIn("INVARIANT:INV-SEC-003:1", {w.target_id for w in selected}) + + def test_an_invariant_impact_already_listed_is_not_duplicated(self) -> None: + # INV-SEC-001 is VIOLATED in the parent and impact lists it: impact's own item stays + refs = self.refs(INV_SEC_001=InvariantStatus.VIOLATED) + selected = DailScopePolicy().select(self.report, self.protected, parent_refs=refs) + self.assertEqual(selected, self.report.verification_work) + + +class TestReasonCodes(unittest.TestCase): + def setUp(self) -> None: + self.report = ib.report() + self.protected = tuple(ib.registry().values()) + + def test_condition_added_items_carry_condition_scope(self) -> None: + listed = set(self.report.verification_work) + for policy in (StatefulFullScopePolicy(), StatelessScopePolicy((SEC2,))): + with self.subTest(policy=type(policy).__name__): + added = [w for w in policy.select(self.report, self.protected) if w not in listed] + self.assertTrue(added) + self.assertEqual({w.reason for w in added}, {CONDITION}) + + def test_no_condition_added_item_carries_the_stand_in(self) -> None: + listed = set(self.report.verification_work) + refs = trusted_state({SEC2: InvariantStatus.VIOLATED}).invariant_refs + for policy in (DailScopePolicy(), StatefulFullScopePolicy(), StatelessScopePolicy((SEC3,))): + selected = policy.select(self.report, self.protected, parent_refs=refs) + for item in selected: + if item not in listed: + self.assertIsNot(item.reason, ReasonCode.SCOPE_TYPE_CHANGED) + + +class TestDailEndToEnd(OrchestratorCase): + def setUp(self) -> None: + super().setUp() + self._dirs = tempfile.TemporaryDirectory(prefix="dail-p8a2-") + self.addCleanup(self._dirs.cleanup) + self.root = Path(self._dirs.name) + + def status(self, state, invariant_id: str) -> InvariantStatus: # type: ignore[no-untyped-def] + return next(r for r in state.invariant_refs if r.invariant_id == invariant_id).status + + def test_an_untouched_violated_invariant_is_verified_and_decides(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, resized(self.root, BASELINE_DIR, "resize"))) + assert result.work is not None and result.decision is not None + reasons = {w.target_id: w.reason for w in result.work} + self.assertIs(reasons.get("INVARIANT:INV-SEC-001:1"), NOT_PROTECTED) + self.assertIs(result.kind, ResultKind.REJECTED) + self.assertEqual(result.decision.reason.value, "REQUIRED_CHECK_FAILED") # a fresh FAIL + + def test_the_middle_step_promotes_under_c_and_c_prime(self) -> None: + for name, options in (("C", {}), ("C'", {"identity_strategy": AddressOnlyIdentity()})): + with self.subTest(condition=name): + self.fresh_database() + v0 = self.baseline().state + middle = resized(self.root, BASELINE_DIR, f"middle-{len(name)}") + first = self.evaluate(self.request(v0, middle, open_targets=(SEC1,)), **options) + self.assertIs(first.kind, ResultKind.PROMOTED) + assert first.state is not None + self.assertIs(self.status(first.state, SEC1), InvariantStatus.VIOLATED) + last = self.evaluate(self.request(first.state, PATCH_SAFE_DIR, n=2), **options) + self.assertIs(last.kind, ResultKind.PROMOTED) + assert last.state is not None + self.assertIs(self.status(last.state, SEC1), InvariantStatus.PROTECTED) + + def test_a_protected_parent_gets_exactly_the_impact_work(self) -> None: + v0 = self.baseline(PATCH_SAFE_DIR).state + self.assertTrue(all(r.status is InvariantStatus.PROTECTED for r in v0.invariant_refs)) + result = self.evaluate(self.request(v0, resized(self.root, PATCH_SAFE_DIR, "resize-safe"))) + assert result.work is not None and result.impact is not None + self.assertEqual(result.work, result.impact.verification_work) + self.assertIs(result.kind, ResultKind.PROMOTED) + + def test_a_violated_reference_is_still_never_carried(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, resized(self.root, BASELINE_DIR, "carry"))) + assert result.decision is not None + check = next( + c for c in result.decision.required_checks if c.target_id == "INVARIANT:INV-SEC-001:1" + ) + self.assertNotEqual(str(check.source), "CARRIED_FORWARD") diff --git a/tests/unit/test_p8a_conditions.py b/tests/unit/test_p8a_conditions.py new file mode 100644 index 0000000..241703c --- /dev/null +++ b/tests/unit/test_p8a_conditions.py @@ -0,0 +1,424 @@ +"""The four P8 conditions (P8a; C-198, C-205, C-206, C-20, C-22, C-127, C-142). + +Expected values come from the decided text (C-198): + +A, Stateless: "verification work = the gate items (STRUCTURAL, CONSISTENCY) and the repair target's +invariant item, or the gate items only for change_request scenarios; required set = the gate items +and the repair target only; no ledger use, no carry-forward." +B, Stateful Full: "verification work = the gate items, every invariant on the current state and +every RESOURCE baseline item of the impact work set; required set as stateful_full_policy (every +invariant on the state, carry-forward off)." +C, DAIL: "DailScopePolicy unchanged; dail_policy (carry-forward on)." +C', DAIL without identity evidence: "AddressOnlyIdentity (C-127), DailScopePolicy, as C." + +C-205: "A current state that holds an invariant outside the explicit set is an ESCALATE +(POLICY_REGISTRY_REGRESSION, C-41), so nothing is dropped silently." +C-206: "only with the flag may the set be empty; every existing caller keeps the 'at least +one' rule." +""" + +from __future__ import annotations + +import dataclasses +import inspect +import unittest + +from core.application.baseline import BaselineRequest +from core.application.orchestrator import EvaluationOrchestrator, ResultKind +from core.application.scope_policy import ( + DailScopePolicy, + StatefulFullScopePolicy, + StatelessScopePolicy, + VerificationScopePolicy, +) +from core.domain.enums import DecisionType +from core.domain.errors import DomainValidationError +from core.domain.impact import WorkPriority, WorkTargetType +from core.domain.promotion import DecisionReason +from core.domain.state import TrustedState +from core.identity.strategy import ( + ADDRESS_ONLY_ALGORITHM_VERSION, + DEFAULT_STRATEGY, + IDENTITY_ALGORITHM_VERSION, + AddressOnlyIdentity, + EvidenceIdentity, +) +from core.promotion import policy as policies +from core.promotion.decide import decide +from core.promotion.policy import dail_policy, stateful_full_policy, stateless_policy +from tests import impact_builders as ib +from tests.domain_builders import LINEAGE, NORMALIZATION, analyzing_candidate, canonical_resources +from tests.promotion_builders import ( + ALL, + BASELINE_ITEM, + DECISION_ID, + FAIL, + FUNC1, + GATES, + NOW, + PASS, + SEC1, + SEC2, + SEC3, + impact_for, + invariant_definitions, + record, + report_of, + target, + trusted_state, +) +from tests.promotion_orchestrator_builders import ( + BASELINE_DIR, + PATCH_REGRESSION_DIR, + PATCH_SAFE_DIR, + OrchestratorCase, + bindings, + context_for, + definitions, +) + +D = DecisionType +R = DecisionReason + + +def a_policy(required: tuple[str, ...]): # type: ignore[no-untyped-def] + return stateless_policy( + policy_version="policy-a", + invariants=invariant_definitions().values(), + automatic_enabled=True, + required=required, + ) + + +class TestStatelessPolicy(unittest.TestCase): + """The policy constructor of condition A (C-205).""" + + def test_it_differs_from_stateful_full_only_in_the_required_set(self) -> None: + common = { + "policy_version": "p", + "invariants": invariant_definitions().values(), + "automatic_enabled": True, + } + full = stateful_full_policy(**common) + stateless = stateless_policy(**common, required=(SEC1,)) + self.assertTrue(stateless.explicit_required_set) + self.assertFalse(full.explicit_required_set) + self.assertFalse(stateless.carry_forward_enabled) + self.assertEqual(stateless.extra_required, (SEC1,)) + self.assertEqual( + dataclasses.replace(stateless, explicit_required_set=False, extra_required=()), full + ) + + def test_the_existing_policies_keep_the_state_required_set(self) -> None: + for build in (dail_policy, stateful_full_policy): + with self.subTest(policy=build.__name__): + made = build( + policy_version="p", + invariants=invariant_definitions().values(), + automatic_enabled=True, + ) + self.assertFalse(made.explicit_required_set) + + def test_it_is_versioned(self) -> None: + self.assertEqual(policies.STATELESS_RULES_VERSION, "dail-stateless-1") + + def test_it_refuses_an_id_it_does_not_define(self) -> None: + with self.assertRaises(DomainValidationError): + a_policy(("INV-NEW-001",)) + + +class TestStatelessDecision(unittest.TestCase): + """``decide`` under condition A's policy: gates and the target, nothing else.""" + + def world(self, *, ids: tuple[str, ...], results: dict[str, object]): # type: ignore[no-untyped-def] + trusted = trusted_state(ids=ids) if ids else empty_state() + candidate = analyzing_candidate(trusted, canonical_resources(ssh_open=False, db_path=True)) + affected = tuple(i for i in ids if i == SEC1) + impact = impact_for(trusted, candidate, carry=(), affected=affected) + definitions = invariant_definitions() + items = [] + for n, (tid, value) in enumerate(sorted(results.items())): + definition = definitions[tid.split(":")[1]] if tid.startswith("INVARIANT:") else None + items.append(record(candidate, tid, value, definition=definition, n=n)) # type: ignore[arg-type] + return trusted, candidate, impact, report_of(candidate, items) + + def decide(self, world, required: tuple[str, ...]): # type: ignore[no-untyped-def] + trusted, candidate, impact, report = world + return decide( + candidate, trusted, impact, report, a_policy(required), decision_id=DECISION_ID, now=NOW + ) + + def test_the_target_and_the_gates_are_the_whole_required_set(self) -> None: + world = self.world(ids=(SEC1,), results={g: PASS for g in GATES} | {target(SEC1): PASS}) + outcome = self.decide(world, (SEC1,)) + self.assertIs(outcome.decision.decision, D.PROMOTE) + self.assertEqual( + sorted(c.target_id for c in outcome.decision.required_checks), + sorted([*GATES, target(SEC1)]), + ) + + def test_a_missing_baseline_item_is_not_required(self) -> None: + # the impact report owes the baseline item BASELINE_ITEM; condition A does not require it + world = self.world(ids=(SEC1,), results={g: PASS for g in GATES} | {target(SEC1): PASS}) + self.assertIn(BASELINE_ITEM, {w.target_id for w in world[2].verification_work}) + self.assertIs(self.decide(world, (SEC1,)).decision.decision, D.PROMOTE) + # the same world under the DAIL policy requires it + trusted, candidate, impact, report = world + dail = dail_policy( + policy_version="p", invariants=invariant_definitions().values(), automatic_enabled=True + ) + self.assertIs( + decide( + candidate, trusted, impact, report, dail, decision_id=DECISION_ID, now=NOW + ).decision.decision, + D.REJECT, + ) + + def test_an_invariant_outside_the_set_does_not_decide(self) -> None: + results = {g: PASS for g in GATES} | {target(SEC1): PASS, target(FUNC1): FAIL} + outcome = self.decide(self.world(ids=(SEC1,), results=results), (SEC1,)) + self.assertIs(outcome.decision.decision, D.PROMOTE) + + def test_a_fail_of_the_target_rejects(self) -> None: + results = {g: PASS for g in GATES} | {target(SEC1): FAIL} + outcome = self.decide(self.world(ids=(SEC1,), results=results), (SEC1,)) + self.assertIs(outcome.decision.decision, D.REJECT) + self.assertIs(outcome.decision.reason, R.REQUIRED_CHECK_FAILED) + + def test_a_state_invariant_outside_the_set_escalates(self) -> None: + results = {g: PASS for g in GATES} | {target(SEC1): PASS} + outcome = self.decide(self.world(ids=ALL, results=results), (SEC1,)) + self.assertIs(outcome.decision.decision, D.ESCALATE) + self.assertIs(outcome.decision.reason, R.POLICY_REGISTRY_REGRESSION) + named = {c.target for c in outcome.decision.blocking_conditions} + self.assertTrue({target(SEC2), target(SEC3), target(FUNC1)} <= named) + + def test_gates_only_on_a_state_with_no_invariant(self) -> None: + outcome = self.decide(self.world(ids=(), results={g: PASS for g in GATES}), ()) + self.assertIs(outcome.decision.decision, D.PROMOTE) + self.assertEqual( + sorted(c.target_id for c in outcome.decision.required_checks), sorted(GATES) + ) + + def test_a_gate_failure_still_rejects(self) -> None: + outcome = self.decide( + self.world(ids=(), results={"STRUCTURAL": FAIL, "CONSISTENCY": PASS}), () + ) + self.assertIs(outcome.decision.decision, D.REJECT) + + +def empty_state() -> TrustedState: + return TrustedState.establish_baseline( + lineage_id=LINEAGE, + resources=canonical_resources(ssh_open=True, db_path=True), + invariant_proofs=[], + evidence_refs=[], + normalization_version=NORMALIZATION, + invariant_registry_version=1, + now=NOW, + allow_no_invariants=True, + ) + + +class TestEmptyBaselineOptIn(unittest.TestCase): + """C-206: an empty invariant set only with the flag.""" + + def test_the_domain_still_refuses_an_empty_baseline_by_default(self) -> None: + with self.assertRaises(DomainValidationError): + TrustedState.establish_baseline( + lineage_id=LINEAGE, + resources=canonical_resources(ssh_open=True, db_path=True), + invariant_proofs=[], + evidence_refs=[], + normalization_version=NORMALIZATION, + invariant_registry_version=1, + now=NOW, + ) + + def test_with_the_flag_the_state_holds_no_invariant(self) -> None: + self.assertEqual(empty_state().invariant_refs, ()) + + def test_the_flag_defaults_to_off(self) -> None: + signature = inspect.signature(TrustedState.establish_baseline) + self.assertIs(signature.parameters["allow_no_invariants"].default, False) + self.assertIs( + {f.name: f for f in dataclasses.fields(BaselineRequest)}["allow_no_invariants"].default, + False, + ) + + def test_a_baseline_request_without_definitions_needs_the_flag(self) -> None: + settings = { + "baseline_id": "00000000-0000-4000-8000-000000003100", + "lineage_id": LINEAGE, + "baseline_dir": BASELINE_DIR, + "variables": None, + "definitions": (), + "bindings": (), + "context": context_for(0), + "now": NOW, + "eligible": True, + } + with self.assertRaises(DomainValidationError): + BaselineRequest(**settings) # type: ignore[arg-type] + self.assertEqual(BaselineRequest(**settings, allow_no_invariants=True).definitions, ()) # type: ignore[arg-type] + + +class TestScopePolicies(unittest.TestCase): + def setUp(self) -> None: + self.report = ib.report() # SEC-001 affected, FUNC uncertain, a baseline item + self.protected = tuple(ib.registry().values()) + + def test_every_policy_has_select_and_a_version(self) -> None: + for chosen in (DailScopePolicy(), StatefulFullScopePolicy(), StatelessScopePolicy(())): + with self.subTest(policy=type(chosen).__name__): + self.assertIsInstance(chosen, VerificationScopePolicy) + self.assertTrue(chosen.version) + self.assertEqual( + len( + { + DailScopePolicy.version, + StatefulFullScopePolicy.version, + StatelessScopePolicy.version, + } + ), + 3, + ) + + def test_dail_is_unchanged(self) -> None: + self.assertEqual( + DailScopePolicy().select(self.report, self.protected), self.report.verification_work + ) + + def test_stateful_full_adds_every_protected_invariant(self) -> None: + work = StatefulFullScopePolicy().select(self.report, self.protected) + targets = {w.target_id for w in work} + self.assertEqual( + targets, + {f"INVARIANT:{d.invariant_id}:{d.version}" for d in self.protected} + | {"RESOURCE:aws_internet_gateway.new"}, + ) + for item in self.report.verification_work: # the impact items are kept as they are + self.assertIn(item, work) + for item in work: + self.assertIs(item.priority, WorkPriority.BLOCKING) + self.assertEqual(len(targets), len(work)) + + def test_stateless_runs_the_target_only(self) -> None: + work = StatelessScopePolicy((SEC1,)).select(self.report, self.protected) + self.assertEqual([w.target_id for w in work], ["INVARIANT:INV-SEC-001:1"]) + self.assertIn(work[0], self.report.verification_work) # impact's own item + + def test_stateless_adds_an_unaffected_target(self) -> None: + work = StatelessScopePolicy((SEC2,)).select(self.report, self.protected) + self.assertEqual([w.target_id for w in work], ["INVARIANT:INV-SEC-002:1"]) + self.assertIs(work[0].target_type, WorkTargetType.INVARIANT) + self.assertNotIn(work[0], self.report.verification_work) + + def test_stateless_with_no_target_runs_nothing_but_the_gates(self) -> None: + self.assertEqual(StatelessScopePolicy(()).select(self.report, self.protected), ()) + + def test_stateless_refuses_a_target_it_has_no_definition_for(self) -> None: + with self.assertRaises(DomainValidationError): + StatelessScopePolicy(("INV-NEW-001",)).select(self.report, self.protected) + + +class TestIdentitySeam(unittest.TestCase): + def test_the_production_default_is_evidence_identity(self) -> None: + default = inspect.signature(EvaluationOrchestrator.__init__).parameters["identity_strategy"] + self.assertIs(default.default, DEFAULT_STRATEGY) + self.assertIsInstance(DEFAULT_STRATEGY, EvidenceIdentity) + + +class TestConditionsEndToEnd(OrchestratorCase): + """The real orchestrator over a database, one condition at a time (canonical fixtures).""" + + def test_c_prime_reports_the_address_only_identity(self) -> None: + v0 = self.baseline().state + result = self.evaluate( + self.request(v0, PATCH_SAFE_DIR), identity_strategy=AddressOnlyIdentity() + ) + assert result.identity is not None + self.assertEqual(result.identity.algorithm_version, ADDRESS_ONLY_ALGORITHM_VERSION) + self.assertIs(result.kind, ResultKind.PROMOTED) + + def test_c_reports_the_evidence_identity_by_default(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_SAFE_DIR)) + assert result.identity is not None and result.work is not None + self.assertEqual(result.identity.algorithm_version, IDENTITY_ALGORITHM_VERSION) + + def test_b_verifies_every_invariant(self) -> None: + v0 = self.baseline().state + result = self.evaluate( + self.request(v0, PATCH_SAFE_DIR), + scope_policy=StatefulFullScopePolicy(), + policy_factory=stateful_full_policy, + ) + self.assertIs(result.kind, ResultKind.PROMOTED) + assert result.verification is not None and result.decision is not None + verified = {r.target_id for r in result.verification.results} + for d in definitions(): + self.assertIn(f"INVARIANT:{d.invariant_id}:{d.version}", verified) + self.assertTrue(all(c.source is not None for c in result.decision.required_checks)) + self.assertNotIn( + "CARRIED_FORWARD", {str(c.source) for c in result.decision.required_checks} + ) + + def test_c_carries_unaffected_invariants_forward(self) -> None: + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_SAFE_DIR)) + assert result.decision is not None + self.assertIn("CARRIED_FORWARD", {str(c.source) for c in result.decision.required_checks}) + + def test_a_checks_only_the_target_and_promotes_a_regression(self) -> None: + # the regression closes SSH and breaks EC2 -> RDS; A does not require INV-FUNC-001 + sec1 = next(d for d in definitions() if d.invariant_id == SEC1) + v0 = self.baseline(definitions=(sec1,), bindings=()).state + self.assertEqual([r.invariant_id for r in v0.invariant_refs], [SEC1]) + result = self.evaluate( + self.request(v0, PATCH_REGRESSION_DIR), + scope_policy=StatelessScopePolicy((SEC1,)), + policy_factory=lambda **kw: stateless_policy(**kw, required=(SEC1,)), + ) + self.assertIs(result.kind, ResultKind.PROMOTED) + assert result.verification is not None + self.assertEqual( + sorted(r.target_id for r in result.verification.results), + sorted([*GATES, target(SEC1)]), + ) + + def test_the_same_regression_is_rejected_under_b_and_c(self) -> None: + for name, options in ( + ("C", {}), + ( + "B", + {"scope_policy": StatefulFullScopePolicy(), "policy_factory": stateful_full_policy}, + ), + ): + with self.subTest(condition=name): + self.fresh_database() + v0 = self.baseline().state + result = self.evaluate(self.request(v0, PATCH_REGRESSION_DIR), **options) + self.assertIs(result.kind, ResultKind.REJECTED) + + def test_a_on_a_change_request_runs_the_gates_only(self) -> None: + v0 = self.baseline( + PATCH_SAFE_DIR, definitions=(), bindings=(), allow_no_invariants=True + ).state + self.assertEqual(v0.invariant_refs, ()) + result = self.evaluate( + self.request(v0, PATCH_SAFE_DIR), + scope_policy=StatelessScopePolicy(()), + policy_factory=lambda **kw: stateless_policy(**kw, required=()), + ) + self.assertIs(result.kind, ResultKind.PROMOTED) + assert result.verification is not None and result.state is not None + self.assertEqual(sorted(r.target_id for r in result.verification.results), sorted(GATES)) + self.assertEqual(result.state.invariant_refs, ()) + + def test_bindings_are_unchanged_for_the_other_conditions(self) -> None: + self.assertEqual(len(bindings()), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_p8a_open_targets.py b/tests/unit/test_p8a_open_targets.py new file mode 100644 index 0000000..a2dcd11 --- /dev/null +++ b/tests/unit/test_p8a_open_targets.py @@ -0,0 +1,229 @@ +"""Open repair targets in ``decide`` (P8a; C-196). Expected values come from the decided text: + +"For an open target, a FAIL result on the candidate does not block promotion, and the invariant +stays VIOLATED in the new trusted state with the fresh FAIL evidence. Every other required invariant +is decided exactly as before. An open target must be VIOLATED in the parent (the request is refused +otherwise), so the rule can never hide a regression of a PROTECTED invariant. It still needs a fresh +verification result: UNKNOWN, UNSUPPORTED or VERIFIER_ERROR on it blocks as before, and a missing +result blocks. ... The default is no open target, so every existing behaviour, including Doc 15 +§13.2 'SEC FAIL + FUNC PASS -> REJECT', is unchanged. The open targets of each transition are +recorded in its decision evidence." + +The world is the canonical one of ``tests.promotion_builders``: INV-SEC-001 is VIOLATED in the +parent, the other three invariants are PROTECTED. +""" + +from __future__ import annotations + +import unittest +from typing import Any + +from core.domain.enums import DecisionType, VerificationResult +from core.domain.promotion import ( + DecisionReason, + OracleVerdict, + PromotionEvidence, + PromotionInputError, + ProofSource, +) +from core.promotion.decide import check_open_targets, decide +from tests.promotion_builders import ( + DECISION_ID, + ERROR, + FAIL, + FUNC1, + NOW, + PASS, + SEC1, + SEC2, + UNKNOWN, + UNSUPPORTED, + World, + make_world, + record, + run, + target, +) + +D = DecisionType +R = DecisionReason + + +def decide_open(world: World, open_targets: Any, **options: Any): # type: ignore[no-untyped-def] + return decide( + world.candidate, + world.trusted, + world.impact, + world.verification, + world.policy, + decision_id=options.pop("decision_id", DECISION_ID), + now=NOW, + open_targets=open_targets, + **options, + ) + + +class TestFailOnAnOpenTarget(unittest.TestCase): + def test_a_fail_on_an_open_target_promotes(self) -> None: + world = make_world(results={target(SEC1): FAIL}) + outcome = decide_open(world, (SEC1,)) + self.assertIs(outcome.decision.decision, D.PROMOTE) + self.assertIs(outcome.decision.reason, R.ALL_REQUIRED_PROVEN) + + def test_without_the_open_target_the_same_fail_rejects(self) -> None: + # Doc 15 §13.2 "SEC FAIL + FUNC PASS -> REJECT" is unchanged by default + world = make_world(results={target(SEC1): FAIL}) + for outcome in (run(world), decide_open(world, ())): + with self.subTest(outcome=outcome.decision.decision_id): + self.assertIs(outcome.decision.decision, D.REJECT) + self.assertIs(outcome.decision.reason, R.REQUIRED_CHECK_FAILED) + + def test_the_fail_is_the_fresh_evidence_of_the_open_target(self) -> None: + world = make_world(results={target(SEC1): FAIL}) + failed = world.records[target(SEC1)] + outcome = decide_open(world, (SEC1,)) + check = next(c for c in outcome.decision.required_checks if c.target_id == target(SEC1)) + self.assertIs(check.source, ProofSource.VERIFIED) + self.assertEqual(check.verification_id, failed.verification_id) + self.assertIn(failed.verification_id, outcome.decision.supporting_results) + entry = next(e for e in outcome.proof_plan if e.invariant_id == SEC1) + self.assertIs(entry.source, ProofSource.VERIFIED) + self.assertEqual(entry.verification_id, failed.verification_id) + + def test_a_pass_on_an_open_target_still_promotes(self) -> None: + world = make_world() + outcome = decide_open(world, (SEC1,)) + self.assertIs(outcome.decision.decision, D.PROMOTE) + + def test_the_open_targets_are_recorded_in_the_decision_evidence(self) -> None: + world = make_world(results={target(SEC1): FAIL}) + outcome = decide_open(world, (SEC1,)) + self.assertEqual(outcome.evidence.open_targets, (SEC1,)) + self.assertEqual(outcome.evidence.to_dict()["open_targets"], [SEC1]) + again = PromotionEvidence.from_dict(outcome.evidence.to_dict()) + self.assertEqual(again, outcome.evidence) + + def test_no_open_target_leaves_the_evidence_record_as_it_was(self) -> None: + # the field is absent from the record (and from its hash) when there is no open target + outcome = run(make_world()) + self.assertEqual(outcome.evidence.open_targets, ()) + self.assertNotIn("open_targets", outcome.evidence.to_dict()) + self.assertEqual(PromotionEvidence.from_dict(outcome.evidence.to_dict()), outcome.evidence) + + def test_an_insufficient_fail_on_an_open_target_does_not_promote(self) -> None: + # the FAIL must come from the definition's verifier, like a PASS + world = make_world() + definition = world.definitions[SEC1] + wrong = record( + world.candidate, target(SEC1), FAIL, definition=definition, verifier=("x", "9"), n=50 + ) + world = make_world(results={target(SEC1): wrong}) + outcome = decide_open(world, (SEC1,)) + self.assertIsNot(outcome.decision.decision, D.PROMOTE) + + def test_a_fresh_but_insufficient_fail_does_not_promote(self) -> None: + # fresh (same candidate, invariant and verifier version) but another verifier id, or a + # level below L1: not sufficient evidence (mutation M5 survived the stale case above) + world = make_world() + definition = world.definitions[SEC1] + for n, options in enumerate( + ({"verifier": ("other.verifier", definition.verifier_version)}, {"level": "L0"}) + ): + with self.subTest(options=options): + weak = record( + world.candidate, target(SEC1), FAIL, definition=definition, n=60 + n, **options + ) + outcome = decide_open(make_world(results={target(SEC1): weak}), (SEC1,)) + self.assertIs(outcome.decision.decision, D.REJECT) + self.assertIs(outcome.decision.reason, R.EVIDENCE_NOT_SUFFICIENT) + + def test_an_oracle_fail_on_an_open_target_is_not_a_conflict(self) -> None: + world = make_world(results={target(SEC1): FAIL}) + verdict = OracleVerdict(SEC1, 1, VerificationResult.FAIL, "oracle-1.0") + outcome = decide_open(world, (SEC1,), oracle_verdicts=(verdict,)) + self.assertIs(outcome.decision.decision, D.PROMOTE) + + +class TestOpenTargetsStillBlock(unittest.TestCase): + def test_a_non_success_on_an_open_target_blocks_as_before(self) -> None: + for result, decision, reason in ( + (UNKNOWN, D.RETRY, R.REQUIRED_UNKNOWN), + (UNSUPPORTED, D.REJECT, R.REQUIRED_UNSUPPORTED), + (ERROR, D.ESCALATE, R.VERIFIER_ERROR), + ): + with self.subTest(result=result.value): + world = make_world(results={target(SEC1): result}) + outcome = decide_open(world, (SEC1,)) + self.assertIs(outcome.decision.decision, decision) + self.assertIs(outcome.decision.reason, reason) + self.assertEqual(outcome.decision.decision, run(world).decision.decision) + + def test_a_missing_result_on_an_open_target_blocks(self) -> None: + world = make_world(results={target(SEC1): None}) + outcome = decide_open(world, (SEC1,)) + self.assertIs(outcome.decision.decision, D.REJECT) + self.assertIs(outcome.decision.reason, R.INSUFFICIENT_EVIDENCE) + + def test_a_protected_invariant_that_fails_still_rejects(self) -> None: + world = make_world(results={target(SEC1): FAIL, target(FUNC1): FAIL}) + outcome = decide_open(world, (SEC1,)) + self.assertIs(outcome.decision.decision, D.REJECT) + self.assertIs(outcome.decision.reason, R.REQUIRED_CHECK_FAILED) + self.assertIn(target(FUNC1), {c.target for c in outcome.decision.blocking_conditions}) + self.assertNotIn(target(SEC1), {c.target for c in outcome.decision.blocking_conditions}) + + def test_the_gates_still_block(self) -> None: + world = make_world(results={target(SEC1): FAIL, "STRUCTURAL": FAIL}) + outcome = decide_open(world, (SEC1,)) + self.assertIs(outcome.decision.decision, D.REJECT) + self.assertIs(outcome.decision.reason, R.STRUCTURAL_FAILURE) + + +class TestRefusedOpenTargets(unittest.TestCase): + def test_a_target_that_is_not_violated_in_the_parent_is_refused(self) -> None: + world = make_world(results={target(FUNC1): FAIL}) + for name in (FUNC1, SEC2): # PROTECTED in the parent + with self.subTest(target=name), self.assertRaises(PromotionInputError): + decide_open(world, (name,)) + with self.subTest(target=name, check="direct"), self.assertRaises(PromotionInputError): + check_open_targets(world.trusted, world.policy, (name,)) + + def test_a_target_that_is_not_on_the_parent_is_refused(self) -> None: + world = make_world() + for bad in (("INV-NEW-001",), ("",), (SEC1, SEC1), "INV-SEC-001"): + with self.subTest(open_targets=bad), self.assertRaises(PromotionInputError): + decide_open(world, bad) + + def test_a_violated_target_is_accepted(self) -> None: + world = make_world() + self.assertEqual(check_open_targets(world.trusted, world.policy, (SEC1,)), (SEC1,)) + self.assertEqual(check_open_targets(world.trusted, world.policy, ()), ()) + + +class TestDefaultUnchanged(unittest.TestCase): + def test_empty_open_targets_reproduce_every_decision(self) -> None: + worlds = [ + make_world(), + make_world(results={target(SEC1): FAIL}), + make_world(results={target(FUNC1): FAIL}), + make_world(results={target(SEC1): UNKNOWN}), + make_world(results={target(SEC1): None}), + make_world(results={"CONSISTENCY": FAIL}), + make_world(automatic_enabled=False), + make_world(carry=()), + ] + for n, world in enumerate(worlds): + with self.subTest(world=n): + before = run(world) + after = decide_open(world, ()) + self.assertEqual(after.decision.content_hash, before.decision.content_hash) + self.assertEqual(after.evidence.content_hash, before.evidence.content_hash) + self.assertEqual(after.proof_plan, before.proof_plan) + + def test_pass_and_default_give_the_same_record(self) -> None: + world = make_world(results={target(SEC1): PASS}) + self.assertEqual(run(world).decision.to_dict(), decide_open(world, ()).decision.to_dict()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_p8a_open_targets_e2e.py b/tests/unit/test_p8a_open_targets_e2e.py new file mode 100644 index 0000000..bff11a7 --- /dev/null +++ b/tests/unit/test_p8a_open_targets_e2e.py @@ -0,0 +1,118 @@ +"""Open repair targets through the real orchestrator, commit and database (P8a; C-196). + +"An open target ... stays VIOLATED in the new trusted state with the fresh FAIL evidence ... The +last step has no open target, so the target must PASS there ... An open target must be VIOLATED in +the parent (the request is refused otherwise)." The parent is the canonical baseline (INV-SEC-001 +VIOLATED: public SSH is open; the other three PROTECTED). The middle step is the baseline with the +application security group's description changed: INV-SEC-001 is affected and still fails. +""" + +from __future__ import annotations + +import shutil +import tempfile +from pathlib import Path + +from core.application.orchestrator import ResultKind +from core.domain.enums import InvariantStatus +from core.domain.promotion import PromotionInputError +from tests.promotion_orchestrator_builders import ( + BASELINE_DIR, + PATCH_REGRESSION_DIR, + PATCH_SAFE_DIR, + OrchestratorCase, +) + +SEC1, FUNC1 = "INV-SEC-001", "INV-FUNC-001" + + +def middle_step(root: Path) -> Path: + """The baseline with the description of aws_security_group.app changed (SSH still open).""" + target = root / "middle" + shutil.copytree(BASELINE_DIR, target, ignore=shutil.ignore_patterns("expected.json")) + main = target / "main.tf" + text = main.read_text(encoding="utf-8") + old = 'description = "Application security group"' + assert text.count(old) == 1 + main.write_text( + text.replace(old, 'description = "Application security group, step 1"'), "utf-8" + ) + return target + + +class TestOpenTargetsEndToEnd(OrchestratorCase): + def setUp(self) -> None: + super().setUp() + self._dirs = tempfile.TemporaryDirectory(prefix="dail-p8a-") + self.addCleanup(self._dirs.cleanup) + self.middle = middle_step(Path(self._dirs.name)) + self.v0 = self.baseline().state + + def status(self, state, invariant_id: str) -> InvariantStatus: # type: ignore[no-untyped-def] + return next(r for r in state.invariant_refs if r.invariant_id == invariant_id).status + + def test_without_an_open_target_the_middle_step_is_rejected(self) -> None: + result = self.evaluate(self.request(self.v0, self.middle)) + self.assertIs(result.kind, ResultKind.REJECTED) + self.assertEqual(self.current().state_id, self.v0.state_id) + + def test_the_middle_step_promotes_with_the_target_still_violated(self) -> None: + result = self.evaluate(self.request(self.v0, self.middle, open_targets=(SEC1,))) + self.assertIs(result.kind, ResultKind.PROMOTED) + assert result.state is not None and result.verification is not None + self.assertIs(self.status(result.state, SEC1), InvariantStatus.VIOLATED) + for other in ("INV-SEC-002", "INV-SEC-003", FUNC1): + self.assertIs(self.status(result.state, other), InvariantStatus.PROTECTED) + failed = next( + r for r in result.verification.results if r.target_id == "INVARIANT:INV-SEC-001:1" + ) + assert result.evidence_ids is not None + ref = next(r for r in result.state.invariant_refs if r.invariant_id == SEC1) + self.assertIn(result.evidence_ids[failed.verification_id], ref.evidence_ids) + assert result.outcome is not None + self.assertEqual(result.outcome.evidence.open_targets, (SEC1,)) + self.assertEqual(self.current().state_id, result.state.state_id) + + def test_the_last_step_then_needs_a_pass(self) -> None: + middle = self.evaluate(self.request(self.v0, self.middle, open_targets=(SEC1,))) + assert middle.state is not None + last = self.evaluate(self.request(middle.state, PATCH_SAFE_DIR, n=2)) + self.assertIs(last.kind, ResultKind.PROMOTED) + assert last.state is not None + self.assertIs(self.status(last.state, SEC1), InvariantStatus.PROTECTED) + + def test_a_regression_of_a_protected_invariant_still_rejects(self) -> None: + result = self.evaluate(self.request(self.v0, PATCH_REGRESSION_DIR, open_targets=(SEC1,))) + self.assertIs(result.kind, ResultKind.REJECTED) + self.assertEqual(self.current().state_id, self.v0.state_id) + + def test_an_open_target_with_no_fresh_result_blocks(self) -> None: + # C-196 "a missing result blocks". Since C-210 DailScopePolicy re-checks every VIOLATED + # invariant, so a scope policy that schedules impact's work only (the work set before + # C-210) is used here to leave the target without a result: VIOLATED, it cannot carry. + class ImpactOnly: + def select(self, report, protected, **_): # type: ignore[no-untyped-def] + del protected + return report.verification_work + + result = self.evaluate( + self.request(self.v0, BASELINE_DIR, open_targets=(SEC1,)), scope_policy=ImpactOnly() + ) + self.assertIs(result.kind, ResultKind.REJECTED) + assert result.decision is not None + self.assertEqual(result.decision.reason.value, "INSUFFICIENT_EVIDENCE") + + def test_an_unchanged_open_target_is_rechecked_by_dail(self) -> None: + # C-210: nothing reaches INV-SEC-001, but it is VIOLATED in the parent, so DAIL checks it; + # its fresh FAIL keeps it VIOLATED and the open target does not block (C-196) + result = self.evaluate(self.request(self.v0, BASELINE_DIR, open_targets=(SEC1,))) + self.assertIs(result.kind, ResultKind.PROMOTED) + assert result.state is not None + self.assertIs(self.status(result.state, SEC1), InvariantStatus.VIOLATED) + + def test_a_protected_target_is_refused_before_anything_is_stored(self) -> None: + with self.assertRaises(PromotionInputError): + self.evaluate(self.request(self.v0, self.middle, open_targets=(FUNC1,))) + with self.uow() as u: + self.assertIsNone(u.candidates.get(self.request(self.v0, self.middle).candidate_id)) + self.assertEqual(self.current().state_id, self.v0.state_id)