From 5746f22890394d153acafa0143d2b37336628878 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 00:00:15 +0900 Subject: [PATCH 01/24] feat(management): stage one-use recovery epoch persistence (#2824) Add deployment-fixed, non-reusable persistence and native component regressions. Document original-history and containment prerequisites; no recovery issuer or grant is enabled. --- .../management-recovery-epoch-design.md | 324 ++++++++++++++ .../management/recovery_epoch_state.py | 306 ++++++++++++++ tests/test_management_recovery_epoch_state.py | 398 ++++++++++++++++++ 3 files changed, 1028 insertions(+) create mode 100644 docs/client-features/management-recovery-epoch-design.md create mode 100644 src/auto_coder/management/recovery_epoch_state.py create mode 100644 tests/test_management_recovery_epoch_state.py diff --git a/docs/client-features/management-recovery-epoch-design.md b/docs/client-features/management-recovery-epoch-design.md new file mode 100644 index 000000000..26ac30d9b --- /dev/null +++ b/docs/client-features/management-recovery-epoch-design.md @@ -0,0 +1,324 @@ +# Management recovery epoch integration design + +Issue [#2824](https://github.com/kitamura-tetsuo/auto-coder/issues/2824) owns a +new host ownership epoch after continuous exclusion was lost. Its required +result is a current, exact-target, one-use grant composed from independently +verified original history, current containment and retained fences. This +fragment records implementation boundaries found at source +[`e742338b3f311e6d30a22a8f5341b8072513d813`](https://github.com/kitamura-tetsuo/auto-coder/commit/e742338b3f311e6d30a22a8f5341b8072513d813). +It is a design and prerequisite audit, not a recovery procedure or evidence +that a production grant can currently be issued. + +The bounded durable-state prototype accompanying this design does not wire a +production issuer or authorize recovery. In particular, an unused state API, +component test, operator inventory or saved prerequisite verdict cannot satisfy +the full Issue. Existing unknown-outcome refusals remain correct. No original +claim, missing journal, operation key, target root or initialization outcome is +repaired or reset by this design. + +## Existing authoritative producers + +The prerequisite implementations have useful, distinct contracts: + +| Producer and verifier | Existing fact | Limit relevant to the recovery issuer | +| --- | --- | --- | +| `original_attempts.enroll`, `capture_claim`, `grant_boundary`, `read_frontier` | Protected prospective attempt anchors and a serialized, append-only history frontier. | Enrollment starts a new prospective scope. It does not capture the origin of a previously running baseline deployment. | +| `original_attempt_evidence.derive` and `verify` | Exact original attempt/store bytes and identities, reverse successor enumeration within the recorded scope, current fences, original terminal never-created facts. | The first anchor's old instance is an external baseline. The verifier also refuses original outcomes other than `succeeded`; it does not expose a complete-but-unknown recovery evidence contract. | +| `quarantine_history.load_manifest` and `QuarantineHistory` | Read-only inspection of preserved operation journals and their key/target exclusions. | An inspection manifest is a selection, not complete lineage authority. | +| `containment_observation.observe` and `verify` | Independently repeated installed-route, Docker, source, configuration, kernel, listener and enrollment observations for one supported manager/proxy pair. | It requires the current single-pair profile and an independently held exclusive original owner fence. Additional management-like objects are refused. | +| `fresh_begin.begin_new_instance` and `inspect_new_instance_origin` | A durable, source/root-bound NEW origin before first launch. | `NewInstanceRequest` has no original-history authority binding. NEW original start requires the separately held private-engine profile. | +| `fresh_host_observation.serve_one_fresh_grant` | An authenticated local peer joined to a selected container, original shared fence and supervisor lifetime. | It is the existing fresh protocol, not a deployment-wide recovery epoch issuer, and it has no complete controller/deployment fence-transfer receipt. | + +The declarations above describe actual functions, not replacement interfaces. +The implementation of #2824 must invoke and reverify the applicable original +producers. Copying their serialized output into a new receipt does not enlarge +what that output proves. + +## Demonstrated integration gaps + +### A prospective frontier does not contain its baseline's origin + +`original_attempt_evidence.derive` selects the first anchor's +`(old_instance, old_container_id)` as `baseline`. Later unanchored predecessors +must match that pair, but no corresponding attempt is created or derived for +that baseline. `original_attempts.enroll` does not inspect prior activity and +correctly describes itself as beginning a prospective scope. + +The observable counterexample is an original manager A, an earlier unrecorded +failed successor X, later enrollment L, and a recorded A-to-B attempt. L can +correctly enumerate all attempts captured after its enrollment, but the +unchanged A-to-B frontier contains no evidence deciding whether X existed. +Backward traversal from B and present stopped-state inspection cannot supply +that absent reverse completeness. + +A genuine NEW origin is useful evidence about A, but its current request does +not bind L before A's first-start authority. Joining A's NEW receipt to a later +L enrollment still leaves the uncovered interval. The join must be recorded by +an actual original admission producer, before the earliest relevant launch or +attempt, and enforced by every supported successor admission route. A new +recovery record cannot backfill that event. + +There is a second, separate history distinction. With complete original store +receipts, `derive` still adds an unavailable reason when +`read_fresh_status(..., host=True)` is not `succeeded`, and `verify` refuses any +unavailable reason. A recovery composition cannot discard that refusal text. +If the supported recovery profile includes validly recorded unknown outcomes, +its authoritative evidence contract must distinguish complete original facts +with an unresolved outcome from missing or unverifiable original facts. Both +retain their original outcome; neither becomes a launch permission. Missing +expected stores continue to refuse unless the original producer's affirmative +terminal never-created boundary proves that exact exception. + +### Current containment's owner lock conflicts with required shared custody + +`containment_kernel_subobservation._external_owner_fence` requires an external +holder distinct from the observer and verifies a kernel `WRITE` flock on the +exact original owner file. REQ-104 requires the recovery issuer to hold a newly +acquired independent shared lock on every original owner file while it verifies +containment at the issuance boundary. Those two locks cannot coexist on the +same Linux file. + +An isolated kernel probe using a real exclusive holder and a separate process +that independently opens the same file observed `LOCK_SH | LOCK_NB` refuse +while the exclusive lock was held, then acquire after its release. This is +positive evidence of the composition conflict, not Docker or recovery +acceptance. Duplicating or inheriting the exclusive holder's descriptor would +not establish the independent shared acquisition required by REQ-104. + +A recovery observation profile therefore needs a defined, independently +verifiable shared-fence contract. It must recheck the exact file, kernel owner, +process/boot lifetime and current custody without weakening the existing +exclusive-fence profile or asserting coverage of the earlier gap. + +### Single-pair, source and engine profiles do not currently join + +`containment_observation.observe` refuses a management/proxy-like Docker object +outside its selected pair as `docker-alternative-management`. A recovery +frontier can contain multiple retained ancestor and failed-successor objects; +the selected consumer may also exist when the grant is issued. Filtering those +objects out before calling the verifier would hide exactly the scope that +REQ-101 through REQ-103 require it to establish. + +`installed_bootstrap_source.inspect_installed_bootstrap_source` recognizes two +explicit original bootstrap digests: + +- Ordinary original: `66bcfefa86aa2bd88dd4a253405fc6faac65675917e8d393e587d5a62fff83bb`. +- Quarantined original: `6a7da1f8f57aab76acb062a911598f75f4ea0b237308389f3807bc6368785e59`. + +A matching image name, caller digest, current checked-out source or successful +original-history test is not another recognized source profile. Meanwhile, +`original_attempt_evidence.derive` requires the enrolled source identity to +match `original_attempts.source_identity()` of the verifying package. The +supported set of original source profiles must be explicit; it cannot be +inferred from unrelated component success. + +Finally, `fresh_new_host_start.start_new_docker_once` requires an independently +verified private Docker Engine and its recorded `DOCKER_HOST` socket. Current +containment source and route discovery reject `DOCKER_HOST`/`DOCKER_CONTEXT` +overrides and inspect the local system engine. Running one verifier against the +outer engine and another against an unbound inner engine does not prove a +single deployment's ownership or controller scope. + +## Minimal prerequisite capabilities proposed for review + +These are proposed capability boundaries, not newly published Issues, +authorizations, changes to #2824's Objective, or waivers of its Requirements. +The existing recovery issuer remains responsible for REQ-101 through REQ-108. + +**Prospective deployment genesis and complete lineage bridge.** Establish an +original, protected binding between a never-before-launched deployment origin +and its attempt history before any supported first-start or successor authority +can escape. The producer must carry that binding through ordinary and failed +successors and refuse a missing or advanced frontier. Its independent verifier +must prove the baseline and all recorded successor branches from original +facts, preserving unknown outcomes and distinguishing incomplete facts from +known unresolved outcomes. The output remains evidence-only. Historical +pre-enrollment deployments and unsupported source profiles remain unavailable; +there is no legacy conversion. + +**Current containment for a complete recovery deployment.** Independently +observe the authorized set of original generations, their proxies and all +ordinary controller routes against the same engine/source profile as that +history. Define issuer-held independent SH fences and separately retained +controller/deployment custody, preserve exact current namespace/resource pins, +and repeat the observations in the actual issuer invocation. An additional +container is accepted only when its exact role and original or target binding +are verified; an omitted or unrecognized controller still refuses. This adds +a recovery-specific contract without relaxing ordinary fresh admission or the +quarantined-original distinction. + +These are independent producers. The lineage bridge can produce correct +non-authorizing original evidence without a recovery grant. Current +containment can produce correct non-authorizing observations without issuing +or consuming an epoch. Their state/lifetime and runtime authorities differ. +The host issuer should compose their defined outputs at one durable issuance +boundary rather than silently invent both producers inside a nominal adapter. + +**Unresolved supported-profile decision:** choose the production origin and +engine model for the first supported complete recovery deployment. Two concrete +paths are a prospective genesis path using the local system engine, with its +own explicit closed-controller admission, or extending the genesis/lineage and +containment observers to the existing private-engine NEW origin. The former +changes the supported genesis admission surface; the latter must bind both +outer private-engine custody and inner deployment facts. Existing component +code does not select between them. Neither option authorizes migrating, +re-enrolling or launching the current incident's deployment. + +## Requirement-by-requirement implementation audit + +| Requirement | Reusable implementation | Missing production fact or integration | +| --- | --- | --- | +| REQ-101 | Original history derivation/verification; quarantine journal reader. | Original baseline-to-history bridge, full ancestor scope, and explicit complete-but-unknown evidence semantics. No synthetic baseline or lost-journal conversion. | +| REQ-102 | Approved enrollment, installed source/mount verification, complete route observer. | One accepted source/engine profile covering all lineage generations and the selected consumer. Operator lists remain selections. | +| REQ-103 | Exact original process/cgroup observations, pinned namespace checks, listener and controller inspection. | Recovery-specific current SH/resource-custody observation repeated in the issuer immediately before grant. | +| REQ-104 | `acquire_prior_owner_fence`, protected file opens and `_assert_pid_flock`. | Issuer-owned independent SH acquisition for the exact complete original set, rechecked after acquisition and at issuance. | +| REQ-105 | Protected, exclusive, append-only persistence patterns; bounded recovery state prototype. | Deployment-fixed production inventory and complete evidence binding, then a single reservation/intent/consumption/receipt state machine reached by the real issuer. A prototype alone is not this gate. | +| REQ-106 | Existing local peer credentials, challenge and selected-container/owner checks; immutable READY validators. | Current whole-scope verification, complete consumer fence-custody receipt, durable one-use consumption before dispatch, and current host-lifetime/readiness handling. | +| REQ-107 | Existing old-root unknown-outcome and first-launch refusals. | Explicit new-epoch authorization and original-exclusion composition that cannot reuse an old root, plan, claim or operation key. | +| REQ-108 | Real original-host boundary fixtures, Docker fresh tests and disposable closed-host fixture components. | All integrated AS-B01 through AS-B05 scenarios through the recovery production issuer and actual prerequisite verifiers. Existing component receipts do not satisfy this row. | + +The durable prototype may verify persistence and locking behavior independently. +It does not make any missing row ready, and the absence of an issuer entry point +must not be presented as successful fail-closed completion of the Issue. + +## Issuer and consumer protocol boundary + +A prospective issuer needs a fresh authenticated local challenge bound to the +fixed deployment, explicit authorization, exact target/preparation and exact +consumer process/boot lifetime. The existing `SO_PEERCRED` and container/ledger +joins are useful mechanisms, but the recovery protocol must additionally prove +custody of the complete required fence set. + +Original shared locks require independent acquisition by each relevant holder. +A passed or inherited descriptor must not be reported as that independent +acquisition. Controller/deployment exclusive fences have a different custody +question: if they are transferred using file descriptors, the issuer must wait +for a challenge-bound receipt and independently inspect the exact receiving +process's descriptors and kernel locks before consuming the epoch. A queued +`SCM_RIGHTS` message or successful `sendmsg` is not receipt. A bare caller claim +that it holds a descriptor is insufficient as well. + +The target source/configuration and installed READY must be revalidated at that +same boundary using their actual validators. `ManagementVersionStager.lookup`, +`self_update_handoff.verify_installed_runtime` and `fresh_runtime_binding` are +existing sources to evaluate for the selected runtime profile; their contracts +must be matched, not replaced by the text `READY` or by an image SHA alone. +All original lineage, current containment, target and fence observations must +remain exact across final durable consumption and dispatch. + +Consumption precedes any escaping grant. Lost response, interrupted receipt, +partial publication or host exit leaves the original state preserved and +non-reusable, including for another target/preparation. Current readiness must +become unavailable after host loss without rewriting a historical observation. +An already emitted grant may race the first launch: the testable requirement +is retained custody and no second authorization, not an impossible promise +that no `Popen` can occur after host death. + +## Integrated production-origin test plan + +| Scenario | Production origin and material barrier | Required observed outcome | +| --- | --- | --- | +| AS-B01 / REQ-101, 107, 108 | Genuine prospective genesis followed by A-to-B-to-C and an original refused successor; actual original verifiers inside issuer. Remove/mutate each baseline, anchor, store, expected failed record and attempt binding in separate preserved cases. | Positive complete scope; exact refusal for missing, stale, forged or cross-deployment facts. Genuine original terminal never-created control succeeds independently. No new historical record, cleared unknown outcome or store repair. | +| AS-B02 / REQ-102, 103, 108 | Full selected closed profile in a disposable systemd VM with real Docker, ordinary restart/watch/service/timer/cron/remote routes and a preserved worker. Observe actual cgroup and namespace membership before each action. | Every omitted, enabled or unknown route, escaped/nonleader task and occupied current listener/namespace refuses. Positive exited-task control is independently measured. Worker/configuration and all original stores remain intact. | +| AS-B03 / REQ-104, 106, 108 | Independent real host and consumer processes open exact original files. Pause at an observed acquisition/receive boundary, then have another process attempt target EX while the intended holds overlap. | Independent SH ownership and exact peer lifetime pass. Busy/replaced/missing files, inherited-as-independent locks, omitted/duplicate descriptors, queued-only transfer and wrong receipt refuse before grant. | +| AS-B04 / REQ-105, 107, 108 | Two real production-factory commands use different preparation/target names but one fixed inventory. Pause the first at a recorded reservation boundary and observe the second actually contend before releasing the first. Kill around each record/create/fsync/head boundary. | At most one reservation/consumption path, no alternate-root bypass, no accepted environment/request override, no repair or reuse of a partial/uncertain state. Preserve bytes and identities for independent readback. | +| AS-B05 / REQ-103, 105, 106, 108 | Pause immediately before the actual final grant boundary, mutate source/READY/target/controller/process/fence evidence, then resume. Separately lose replies or kill the host around consumption and dispatch. | Changed/stale input refuses; a consumed epoch never grants again. Host loss makes current readiness unavailable. If a grant already escaped, observe retained consumer custody and no second grant rather than assuming launch cannot race. | + +`tests/fixtures/original_host_boundaries.py` demonstrates suitable barriers at +`before-event`, `after-event`, `after-pending`, `after-replace` and `after-sync` +around actual producer publication. Its subprocesses execute the production +claim/grant path; instrumentation observes or interrupts real operations. +Merely starting two processes concurrently does not establish contention. + +`tests/test_management_fresh_end_to_end.py` supplies genuine Docker histories +and authenticated-child flows. It does not independently establish the full +closed deployment profile. Conversely, +`scripts/test_management_containment_closed_host.py` exercises actual producer +and separate-consumer containment commands, real routes and current kernel +controls, but does not issue a recovery grant or establish complete lineage. +The integrated test must join these authorities through the new issuer. + +The closed-host fixture requires its preexisting protected synthetic-host +authorization, an initially empty KVM/QEMU VM with PID 1 systemd and the initial +host namespaces. It deliberately inhibits fixture host routes. It is not a +command to run on a populated production host. The existing Supervisor +`fresh-host` CI profile runs privileged Docker tests, but does not itself run +this full closed-host recovery scenario. New integrated nodes and their native +receipts must be collected and required by an appropriate isolated profile; +no missing-node or skip allowance may silently convert unavailable authority +into acceptance. + +## Durable state component in this draft + +`recovery_epoch_state.hold_recovery_epoch()` has no arguments. Its production +root is fixed at `/var/lib/auto-coder/recovery-epoch`; environment variables, +request paths, target names and preparation names cannot select another root. +The factory requires the approved Unix owner and protected, canonical directory +ancestors. It does not provision or repair a missing directory, inventory or +lock. An operator would have to separately review and install the inventory; +this draft performs no such operation. + +The finite `EpochInventory` binds the exact root and lock device/inode identities, +deployment, authorization, sorted unique enrollment set, request, preparation, +Docker target, source digest and configuration digest. An `EpochBinding` must +match those fields and additionally binds the verified-evidence digest and the +actual current issuer boot/PID/start lifetime. The component validates those +joins and their exact schema, but cannot establish that the supplied evidence +was independently produced. The missing issuer must establish that fact before +calling it; inventing a digest is not an authorization mechanism. + +An independently opened exclusive flock serializes the whole state invocation. +The component rechecks the current process's actual kernel lock, protected +inventory identity and bytes, every retained record and the complete namespace +before each transition. A forked or released holder, replaced lock/root/record, +changed inventory, conflicting alias or existing phase refuses. + +| Transition | Persisted fact | Reentry behavior | +| --- | --- | --- | +| `reserve(binding)` | `reserved.json` and immutable `reserved-head.json` | Any subsequent invocation refuses this deployment state, including a different target/preparation. | +| `record_intent(binding)` | `intent.json` and immutable `intent-head.json`, linked to the prior head | A partial or uncertain write is preserved and cannot be repaired or resumed. | +| `consume(binding)` | `consumed.json`/head followed by `receipt.json`/head | Returns only after both transitions are durable. A lost return, exception or second call cannot produce another receipt. | + +Each new file uses exclusive creation and no-follow opens, file fsync and +directory fsync. The append heads and records retain the exact binding and +previous-head digest. The library never deletes a partial file, replaces a +committed head, resets a reservation or reconstructs state after restart. +`EpochReceipt.grant_delivered` is always `False`: this component neither has a +transport nor observes a grant. The host issuer must retain and independently +reverify consumer custody and current evidence at its eventual dispatch boundary. + +`_hold_recovery_epoch(root, expected_uid=...)` is an internal component-test seam. +It must not become a production CLI/environment override. The maintained tests +use isolated user-owned files and real process/kernel flocks. They exercise +material contention after a post-reservation barrier, I/O failures around each +record/head fsync, SIGKILL at observable publication barriers, changed bindings, +path replacement and replay refusal. These are bounded persistence tests, not +production-host, Docker, complete-lineage or integrated grant acceptance. +Process interruption tests do not claim simulated storage-power-loss durability. + +Run the maintained component selection with: + +```sh +bash scripts/test.sh --ci-tests-only tests/test_management_recovery_epoch_state.py +``` + +The component uses the existing Loguru configuration with console and rotating +file sinks. Its phase records explicitly carry `grant_delivered=False`; they +never report authenticated readiness or recovery success. The eventual issuer's +production trace integration remains part of the unfinished contract below. + +## Observability and completion boundary + +The unwired persistence/design work adds no production processing origin, +admission grant, success event, worker action or dashboard-owned state. +Consequently it supplies no recovery-success trace and no dashboard claim of +readiness. Wiring the eventual issuer must include its real reservation, +refusal, consumption, dispatch and current-host-loss observations in the +production trace contract and `docs/dashboard-observability.md`, with runnable +tests for those boundaries. + +Do not close #2824, merge an implementation as its completion, or deploy a +recovery issuer until the supported-profile decision and missing producers are +resolved, all eight requirements are actually connected, and the integrated +native evidence passes. Completion of a prototype or this audit is not that +result. diff --git a/src/auto_coder/management/recovery_epoch_state.py b/src/auto_coder/management/recovery_epoch_state.py new file mode 100644 index 000000000..565d7cb09 --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_state.py @@ -0,0 +1,306 @@ +"""Deployment-fixed, single-use recovery-epoch persistence. + +These receipts record only durable state. They are not a grant, containment +proof, historical completeness verdict, or permission to launch. The issuer +must independently verify those facts before consuming this state. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import re +import uuid +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterator + +from ..logger_config import get_logger +from .containment_enrollment import _approved_directory +from .fresh_host_observation import _assert_pid_flock +from .fresh_preflight import ObservedIdentity, _file, _identity, _open +from .original_attempts import canonical, identity + +EPOCH_ROOT = Path("/var/lib/auto-coder/recovery-epoch") +logger = get_logger(__name__) +_PHASES = ("reserved", "intent", "consumed", "receipt") +_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z", re.ASCII) +_SHA256 = re.compile(r"[0-9a-f]{64}\Z", re.ASCII) +_MAX_BYTES = 1024 * 1024 + + +class EpochUnavailable(ValueError): + """An absent, conflicting or uncertain state is never repaired or replayed.""" + + +@dataclass(frozen=True) +class EpochInventory: + schema_version: int + root: str + root_device: int + root_inode: int + lock_device: int + lock_inode: int + deployment_id: str + authorization_id: str + enrollment_ids: tuple[str, ...] + request_id: str + preparation_id: str + target_id: str + source_sha256: str + config_sha256: str + + +@dataclass(frozen=True) +class EpochBinding: + deployment_id: str + authorization_id: str + enrollment_ids: tuple[str, ...] + request_id: str + preparation_id: str + target_id: str + source_sha256: str + config_sha256: str + evidence_sha256: str + host_boot_id: str + host_pid: int + host_start_token: str + + +@dataclass(frozen=True) +class EpochReceipt: + inventory_sha256: str + binding_sha256: str + consumed_sha256: str + receipt_sha256: str + # This state machine never observes dispatch or grants launch authority. + grant_delivered: bool = False + + +def issuer_lifetime() -> tuple[str, int, str]: + """Read the actual current issuer's boot, PID and Linux start token.""" + boot = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() + fields = Path("/proc/self/stat").read_text(encoding="ascii").rsplit(")", 1)[1].split() + if str(uuid.UUID(boot)) != boot or len(fields) < 20 or not fields[19].isascii() or not fields[19].isdecimal() or int(fields[19]) <= 0 or str(int(fields[19])) != fields[19]: + raise EpochUnavailable("recovery issuer lifetime is unavailable") + return boot, os.getpid(), fields[19] + + +def _digest(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _read(path: Path) -> tuple[ObservedIdentity, bytes]: + observed, encoded = _file(path, private=True, limit=_MAX_BYTES) + if identity(path) != (observed.device, observed.inode): + raise EpochUnavailable("recovery epoch file was replaced") + return observed, encoded + + +def _validate_static(value: EpochInventory | EpochBinding) -> None: + for field in ("deployment_id", "authorization_id", "request_id", "preparation_id"): + token = getattr(value, field) + if not isinstance(token, str) or not _IDENTIFIER.fullmatch(token): + raise EpochUnavailable("invalid recovery epoch " + field) + if not isinstance(value.enrollment_ids, tuple) or not value.enrollment_ids or len(value.enrollment_ids) > 256 or any(not isinstance(token, str) or not _IDENTIFIER.fullmatch(token) for token in value.enrollment_ids): + raise EpochUnavailable("invalid recovery epoch enrollments") + if tuple(sorted(set(value.enrollment_ids))) != value.enrollment_ids: + raise EpochUnavailable("recovery epoch enrollments are aliased or unordered") + for field in ("target_id", "source_sha256", "config_sha256"): + digest = getattr(value, field) + if not isinstance(digest, str) or not _SHA256.fullmatch(digest): + raise EpochUnavailable("invalid recovery epoch " + field) + + +def _parse_inventory(encoded: bytes, root: Path, root_id: tuple[int, int], lock_id: tuple[int, int]) -> EpochInventory: + raw = json.loads(encoded) + if not isinstance(raw, dict) or set(raw) != set(EpochInventory.__dataclass_fields__) or canonical(raw) != encoded or not isinstance(raw.get("enrollment_ids"), list): + raise EpochUnavailable("recovery epoch inventory is noncanonical or incomplete") + raw["enrollment_ids"] = tuple(raw["enrollment_ids"]) + inventory = EpochInventory(**raw) + _validate_static(inventory) + if type(inventory.schema_version) is not int or inventory.schema_version != 1: + raise EpochUnavailable("unsupported recovery epoch inventory schema") + if any(type(getattr(inventory, field)) is not int or getattr(inventory, field) < 0 for field in ("root_device", "root_inode", "lock_device", "lock_inode")): + raise EpochUnavailable("invalid recovery epoch inventory identity") + if inventory.root != str(root) or (inventory.root_device, inventory.root_inode) != root_id or (inventory.lock_device, inventory.lock_inode) != lock_id: + raise EpochUnavailable("recovery epoch inventory root or lock differs") + return inventory + + +class RecoveryEpochState: + """One process's held serialization lifetime, never reusable authorization.""" + + def __init__(self, root: Path, directory: int, lock: int, inventory: EpochInventory, observed: ObservedIdentity, encoded: bytes) -> None: + self.inventory = inventory + self.inventory_sha256 = _digest(encoded) + self._root = root + self._directory = directory + self._lock = lock + self._inventory_observed = observed + self._inventory_encoded = encoded + self._lifetime = issuer_lifetime() + self._records: dict[str, tuple[ObservedIdentity, bytes]] = {} + self._binding: EpochBinding | None = None + self._phase = 0 + self._active = True + self._failed = False + + def _check(self) -> None: + if not self._active or self._failed or issuer_lifetime() != self._lifetime: + raise EpochUnavailable("recovery epoch invocation is closed, failed or inherited") + try: + expected_root = (self.inventory.root_device, self.inventory.root_inode) + if identity(self._root, directory=True) != expected_root or (os.fstat(self._directory).st_dev, os.fstat(self._directory).st_ino) != expected_root: + raise EpochUnavailable("recovery epoch root was replaced") + expected_lock = (self.inventory.lock_device, self.inventory.lock_inode) + if identity(self._root / "epoch.lock") != expected_lock or (os.fstat(self._lock).st_dev, os.fstat(self._lock).st_ino) != expected_lock: + raise EpochUnavailable("recovery epoch lock was replaced") + locks = Path(f"/proc/self/fdinfo/{self._lock}").read_text(encoding="ascii").splitlines() + if not any(line.startswith("lock:") and " FLOCK ADVISORY WRITE " in line for line in locks): + raise EpochUnavailable("recovery epoch exclusive lock is not held") + _assert_pid_flock(os.getpid(), self._lock, "WRITE") + if _read(self._root / "inventory.json") != (self._inventory_observed, self._inventory_encoded): + raise EpochUnavailable("recovery epoch inventory changed") + if set(os.listdir(self._directory)) != {"inventory.json", "epoch.lock"} | set(self._records): + raise EpochUnavailable("recovery epoch contains existing, partial or conflicting state") + for name, expected in self._records.items(): + if _read(self._root / name) != expected: + raise EpochUnavailable("recovery epoch record changed") + except (OSError, ValueError, TypeError) as error: + self._failed = True + raise EpochUnavailable("recovery epoch custody unavailable: " + str(error)) from error + + def _require_binding(self, binding: EpochBinding) -> None: + if not isinstance(binding, EpochBinding): + raise EpochUnavailable("typed recovery epoch binding required") + _validate_static(binding) + for field in ("deployment_id", "authorization_id", "enrollment_ids", "request_id", "preparation_id", "target_id", "source_sha256", "config_sha256"): + if getattr(binding, field) != getattr(self.inventory, field): + raise EpochUnavailable("recovery epoch binding differs from fixed inventory: " + field) + if not isinstance(binding.evidence_sha256, str) or not _SHA256.fullmatch(binding.evidence_sha256): + raise EpochUnavailable("invalid independently verified recovery evidence digest") + if type(binding.host_pid) is not int or (binding.host_boot_id, binding.host_pid, binding.host_start_token) != self._lifetime: + raise EpochUnavailable("recovery epoch binding differs from current issuer lifetime") + if self._binding is not None and binding != self._binding: + raise EpochUnavailable("recovery epoch binding changed after reservation") + + def _create(self, name: str, encoded: bytes) -> None: + descriptor = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self._directory) + try: + # Publish even an empty/partial attempt before filling it. A later + # invocation refuses the entire namespace instead of repairing it. + os.fsync(self._directory) + with os.fdopen(os.dup(descriptor), "wb") as stream: + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + written = _identity(self._root / name, descriptor, _digest(encoded)) + os.fsync(self._directory) + observed, actual = _read(self._root / name) + # Retain the inode actually fsynced by this writer. Matching bytes + # in a replacement file cannot inherit that durability observation. + if observed != written or _identity(self._root / name, descriptor, _digest(encoded)) != written or os.fstat(descriptor).st_nlink != 1 or actual != encoded: + raise EpochUnavailable("recovery epoch append identity or readback differs") + self._records[name] = (observed, actual) + finally: + os.close(descriptor) + + def _append(self, phase: str, binding: EpochBinding) -> str: + self._check() + self._require_binding(binding) + if self._phase >= len(_PHASES) or _PHASES[self._phase] != phase: + raise EpochUnavailable("recovery epoch transition is out of order or already consumed") + previous = _digest(self._records[_PHASES[self._phase - 1] + "-head.json"][1]) if self._phase else "" + value = {"schema_version": 1, "phase": phase, "sequence": self._phase, "previous": previous, "inventory_sha256": self.inventory_sha256, "binding": asdict(binding)} + encoded = canonical(value) + try: + self._create(phase + ".json", encoded) + self._check() + head = canonical({"schema_version": 1, "phase": phase, "record_sha256": _digest(encoded), "previous": previous, "inventory_sha256": self.inventory_sha256}) + self._create(phase + "-head.json", head) + self._check() + self._binding = binding + self._phase += 1 + logger.bind(recovery_epoch_phase=phase, request_id=binding.request_id, target_id=binding.target_id, grant_delivered=False).info("Recovery epoch {} durably recorded; no grant delivery observed", phase) + return _digest(encoded) + except BaseException: + # Failure, including cancellation after a successful write, ends + # this state machine. Nothing is deleted or marked retryable. + self._failed = True + raise + + def reserve(self, binding: EpochBinding) -> None: + """Irreversibly reserve this one deployment-wide approved invocation.""" + self._append("reserved", binding) + + def record_intent(self, binding: EpochBinding) -> None: + """Retain the full currently verified binding before final consumption.""" + self._append("intent", binding) + + def consume(self, binding: EpochBinding) -> EpochReceipt: + """Persist consumption and its final receipt before possible dispatch. + + The returned record grants nothing. A caller must retain and verify + actual consumer fence custody and current evidence at its grant boundary. + A lost return value cannot be obtained by another call or invocation. + """ + consumed = self._append("consumed", binding) + receipt = self._append("receipt", binding) + self._check() + return EpochReceipt(self.inventory_sha256, _digest(canonical(asdict(binding))), consumed, receipt) + + +@contextmanager +def _hold_recovery_epoch(root: Path, *, expected_uid: int) -> Iterator[RecoveryEpochState]: + """Internal explicit fixture seam; production never accepts a root input.""" + state: RecoveryEpochState | None = None + if os.geteuid() != expected_uid: + raise EpochUnavailable("recovery epoch requires its approved Unix owner") + try: + root_id = identity(root, directory=True) + with _open(root, directory=True) as directory, _open(root / "epoch.lock") as lock: + if (os.fstat(directory).st_dev, os.fstat(directory).st_ino) != root_id: + raise EpochUnavailable("recovery epoch directory changed while opening") + lock_id = identity(root / "epoch.lock") + if (os.fstat(lock).st_dev, os.fstat(lock).st_ino) != lock_id: + raise EpochUnavailable("recovery epoch lock changed while opening") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise EpochUnavailable("recovery epoch admission contested") from error + observed, encoded = _read(root / "inventory.json") + inventory = _parse_inventory(encoded, root, root_id, lock_id) + state = RecoveryEpochState(root, directory, lock, inventory, observed, encoded) + state._check() + yield state + state._check() + except (OSError, ValueError, TypeError, KeyError) as error: + if isinstance(error, EpochUnavailable): + raise + raise EpochUnavailable("recovery epoch state unavailable: " + str(error)) from error + finally: + if state is not None: + state._active = False + + +@contextmanager +def hold_recovery_epoch() -> Iterator[RecoveryEpochState]: + """Open only the deployment-fixed root; no request/environment override. + + Provisioning the protected inventory and original empty lock is a separate + reviewed operation. Missing state is unavailable, never auto-initialized. + """ + if os.geteuid() != 0: + raise EpochUnavailable("recovery epoch production issuer requires host root") + try: + _approved_directory(EPOCH_ROOT, uid=0) + with _hold_recovery_epoch(EPOCH_ROOT, expected_uid=0) as state: + yield state + except (OSError, ValueError, TypeError) as error: + if isinstance(error, EpochUnavailable): + raise + raise EpochUnavailable("fixed recovery epoch inventory unavailable: " + str(error)) from error diff --git a/tests/test_management_recovery_epoch_state.py b/tests/test_management_recovery_epoch_state.py new file mode 100644 index 000000000..96af82a7a --- /dev/null +++ b/tests/test_management_recovery_epoch_state.py @@ -0,0 +1,398 @@ +"""Native persistence tests, separate from complete host-issuer acceptance. + +Inventories here are isolated state fixtures. They do not establish historical +completeness, independent containment or authorization to launch a supervisor. +""" + +import fcntl +import hashlib +import inspect +import json +import os +import select +import subprocess +import sys +from dataclasses import asdict, replace +from pathlib import Path + +import pytest + +from auto_coder.management import recovery_epoch_state as epoch +from auto_coder.management.original_attempts import canonical + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@pytest.fixture +def _use_real_commands() -> None: + pass + + +@pytest.fixture +def _use_real_sleep() -> None: + pass + + +def _write(path: Path, value: bytes) -> None: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(value) + + +def _inventory(tmp_path: Path) -> tuple[Path, epoch.EpochInventory]: + root = tmp_path / "epoch" + root.mkdir(mode=0o700) + _write(root / "epoch.lock", b"") + directory, lock = root.stat(), (root / "epoch.lock").stat() + inventory = epoch.EpochInventory(1, str(root), directory.st_dev, directory.st_ino, lock.st_dev, lock.st_ino, "deployment-A", "reviewed-A", ("original-A", "original-B"), "request-A", "preparation-A", "a" * 64, "b" * 64, "c" * 64) + _write(root / "inventory.json", canonical(asdict(inventory))) + return root, inventory + + +def _binding(inventory: epoch.EpochInventory) -> epoch.EpochBinding: + boot, pid, start = epoch.issuer_lifetime() + return epoch.EpochBinding(inventory.deployment_id, inventory.authorization_id, inventory.enrollment_ids, inventory.request_id, inventory.preparation_id, inventory.target_id, inventory.source_sha256, inventory.config_sha256, "d" * 64, boot, pid, start) + + +def _snapshot(root: Path) -> dict[str, tuple[int, int, bytes]]: + return {path.name: (path.stat().st_dev, path.stat().st_ino, path.read_bytes()) for path in root.iterdir() if path.is_file()} + + +def _complete(state: epoch.RecoveryEpochState, binding: epoch.EpochBinding) -> epoch.EpochReceipt: + state.reserve(binding) + state.record_intent(binding) + return state.consume(binding) + + +def _line(process: subprocess.Popen[str]) -> str: + assert process.stdout is not None + assert select.select([process.stdout], [], [], 15)[0], "native process did not reach its observation barrier" + return process.stdout.readline().strip() + + +def test_durable_phases_bind_exact_inventory_and_never_reissue(tmp_path): + root, inventory = _inventory(tmp_path) + initial = _snapshot(root) + binding = _binding(inventory) + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + assert state.inventory == inventory + receipt = _complete(state, binding) + assert receipt.grant_delivered is False + assert receipt.inventory_sha256 == hashlib.sha256(initial["inventory.json"][2]).hexdigest() + assert receipt.binding_sha256 == hashlib.sha256(canonical(asdict(binding))).hexdigest() + assert receipt.consumed_sha256 == hashlib.sha256((root / "consumed.json").read_bytes()).hexdigest() + assert receipt.receipt_sha256 == hashlib.sha256((root / "receipt.json").read_bytes()).hexdigest() + with pytest.raises(epoch.EpochUnavailable, match="already consumed"): + state.consume(binding) + retained = _snapshot(root) + assert {"inventory.json", "epoch.lock"} | {phase + suffix for phase in ("reserved", "intent", "consumed", "receipt") for suffix in (".json", "-head.json")} == set(retained) + previous = "" + for sequence, phase in enumerate(("reserved", "intent", "consumed", "receipt")): + record = json.loads(retained[phase + ".json"][2]) + assert record == {"schema_version": 1, "phase": phase, "sequence": sequence, "previous": previous, "inventory_sha256": receipt.inventory_sha256, "binding": json.loads(canonical(asdict(binding)))} + head = json.loads(retained[phase + "-head.json"][2]) + assert head["record_sha256"] == hashlib.sha256(retained[phase + ".json"][2]).hexdigest() + assert head["previous"] == previous + previous = hashlib.sha256(retained[phase + "-head.json"][2]).hexdigest() + with pytest.raises(epoch.EpochUnavailable, match="closed"): + state.reserve(binding) + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a lost receipt became a second invocation") + assert _snapshot(root) == retained + assert {name: retained[name] for name in initial} == initial + + +@pytest.mark.parametrize( + "field,value", + [ + ("deployment_id", "foreign"), + ("authorization_id", "unreviewed"), + ("enrollment_ids", ("different",)), + ("request_id", "other"), + ("preparation_id", "other"), + ("target_id", "e" * 64), + ("source_sha256", "e" * 64), + ("config_sha256", "e" * 64), + ("evidence_sha256", "unknown"), + ("host_pid", True), + ("host_start_token", "1"), + ("host_boot_id", "different"), + ], +) +def test_binding_differences_refuse_before_persistence(tmp_path, field, value): + root, inventory = _inventory(tmp_path) + initial = _snapshot(root) + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + with pytest.raises(epoch.EpochUnavailable): + state.reserve(replace(_binding(inventory), **{field: value})) + assert _snapshot(root) == initial + + +@pytest.mark.parametrize("name", ["inventory.json", "epoch.lock"]) +def test_missing_or_aliased_fixed_files_are_never_recreated(tmp_path, name): + root, _ = _inventory(tmp_path) + path = root / name + os.link(path, tmp_path / "alias") + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a hardlinked fixed file was accepted") + assert _snapshot(root) == before + path.unlink() + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a missing fixed file was synthesized") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("mutation", ["unknown-field", "missing-field", "noncanonical", "wrong-root", "wrong-lock", "boolean-schema", "duplicate-enrollment"]) +def test_fixed_inventory_has_a_finite_exact_schema(tmp_path, mutation): + root, inventory = _inventory(tmp_path) + value = asdict(inventory) + if mutation == "unknown-field": + value["alternate_root"] = str(tmp_path) + elif mutation == "missing-field": + del value["authorization_id"] + elif mutation == "wrong-root": + value["root_inode"] = inventory.root_inode + 1 + elif mutation == "wrong-lock": + value["lock_inode"] = inventory.lock_inode + 1 + elif mutation == "boolean-schema": + value["schema_version"] = True + elif mutation == "duplicate-enrollment": + value["enrollment_ids"] = ["original-A", "original-A"] + encoded = json.dumps(value).encode() if mutation == "noncanonical" else canonical(value) + (root / "inventory.json").write_bytes(encoded) + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("an invalid or aliased inventory was accepted") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("name", ["reserved.json", "reserved-head.json", "intent.json", "consumed.json", "receipt-head.json", "pending.json"]) +def test_partial_or_conflicting_state_is_preserved_and_unavailable(tmp_path, name): + root, _ = _inventory(tmp_path) + _write(root / name, b'{"interrupted":') + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("uncertain state was treated as a fresh epoch") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("changed", ["inventory", "lock", "record", "root", "released"]) +def test_changes_after_reservation_refuse_without_repair(tmp_path, changed): + root, inventory = _inventory(tmp_path) + binding = _binding(inventory) + before: dict[str, tuple[int, int, bytes]] = {} + preserved = root + with pytest.raises(epoch.EpochUnavailable): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + state.reserve(binding) + if changed == "root": + preserved = tmp_path / "detached" + root.rename(preserved) + root.mkdir(mode=0o700) + elif changed == "released": + fcntl.flock(state._lock, fcntl.LOCK_UN) + else: + path = root / {"inventory": "inventory.json", "lock": "epoch.lock", "record": "reserved.json"}[changed] + original = path.read_bytes() + path.unlink() + _write(path, original) + before = _snapshot(preserved) + state.record_intent(binding) + assert _snapshot(preserved) == before + assert "intent.json" not in before + + +@pytest.mark.parametrize("boundary", range(1, 25)) +@pytest.mark.parametrize("fail_after_sync", [False, True]) +def test_every_record_and_head_fsync_failure_preserves_uncertainty(tmp_path, monkeypatch, boundary, fail_after_sync): + root, inventory = _inventory(tmp_path) + original_fsync = os.fsync + count = 0 + + def interrupted(descriptor): + nonlocal count + count += 1 + if count == boundary and not fail_after_sync: + raise OSError("injected epoch persistence interruption") + original_fsync(descriptor) + if count == boundary and fail_after_sync: + raise OSError("injected epoch persistence interruption") + + with monkeypatch.context() as patch: + patch.setattr(os, "fsync", interrupted) + with pytest.raises(epoch.EpochUnavailable, match="injected epoch persistence interruption"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + _complete(state, _binding(inventory)) + assert count == boundary + before = _snapshot(root) + assert "reserved.json" in before + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("an uncertain persistence result was reset") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("phase", ["reserved", "intent", "consumed", "receipt"]) +def test_cancellation_after_durable_publication_poisons_the_same_invocation(tmp_path, phase): + root, inventory = _inventory(tmp_path) + binding = _binding(inventory) + + def interrupt_after_real_publication(message): + if message.record["extra"].get("recovery_epoch_phase") == phase: + raise KeyboardInterrupt("cancel after durable " + phase) + + sink = epoch.logger.add(interrupt_after_real_publication, catch=False) + try: + with pytest.raises(epoch.EpochUnavailable, match="closed, failed or inherited"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + with pytest.raises(KeyboardInterrupt, match="cancel after durable"): + _complete(state, binding) + assert (root / (phase + ".json")).is_file() + assert (root / (phase + "-head.json")).is_file() + before = _snapshot(root) + # Catching cancellation must not resurrect the same in-memory + # holder and complete issuance after an interrupted boundary. + with pytest.raises(epoch.EpochUnavailable, match="closed, failed or inherited"): + state.consume(binding) + assert _snapshot(root) == before + finally: + epoch.logger.remove(sink) + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("canceled publication was reset by another invocation") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("boundary", [3, 6, 9, 12, 15, 18, 21, 24]) +def test_publication_readback_cannot_adopt_a_same_byte_replacement(tmp_path, monkeypatch, boundary): + root, inventory = _inventory(tmp_path) + original_fsync = os.fsync + count = 0 + replaced = None + before: dict[str, tuple[int, int, bytes]] = {} + names = [phase + suffix for phase in ("reserved", "intent", "consumed", "receipt") for suffix in (".json", "-head.json")] + + def replace_after_real_publication(descriptor): + nonlocal count, replaced, before + original_fsync(descriptor) + count += 1 + if count == boundary: + path = root / names[boundary // 3 - 1] + original_inode = path.stat().st_ino + replacement = tmp_path / "unfsynced-copy" + _write(replacement, path.read_bytes()) + replaced = replacement.stat().st_ino + assert replaced != original_inode + replacement.replace(path) + before = _snapshot(root) + + monkeypatch.setattr(os, "fsync", replace_after_real_publication) + with pytest.raises(epoch.EpochUnavailable, match="append identity or readback differs"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + _complete(state, _binding(inventory)) + assert count == boundary and replaced is not None + assert _snapshot(root) == before + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("replaced publication was repaired by another invocation") + assert _snapshot(root) == before + + +def test_two_native_contenders_reach_the_held_deployment_flock(tmp_path): + root, _ = _inventory(tmp_path) + first_code = """ +import os, sys +from pathlib import Path +from auto_coder.management.recovery_epoch_state import _hold_recovery_epoch +from tests.test_management_recovery_epoch_state import _binding +with _hold_recovery_epoch(Path(sys.argv[1]), expected_uid=os.geteuid()) as state: + state.reserve(_binding(state.inventory)) + print('RESERVED_WITH_FLOCK', flush=True) + sys.stdin.readline() +""" + contender_code = """ +import os, sys +from pathlib import Path +from auto_coder.management.recovery_epoch_state import _hold_recovery_epoch, EpochUnavailable +try: + with _hold_recovery_epoch(Path(sys.argv[1]), expected_uid=os.geteuid()): + raise AssertionError('second contender entered') +except EpochUnavailable as error: + print(str(error), flush=True) +""" + first = subprocess.Popen([sys.executable, "-c", first_code, str(root)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert _line(first) == "RESERVED_WITH_FLOCK" + before = _snapshot(root) + second = subprocess.run([sys.executable, "-c", contender_code, str(root)], capture_output=True, text=True, timeout=15, check=True) + assert second.stdout.strip() == "recovery epoch admission contested" + assert first.poll() is None + assert _snapshot(root) == before + finally: + first.kill() + first.communicate(timeout=10) + assert first.returncode == -9 + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("host death erased the deployment reservation") + assert _snapshot(root) == before + + +@pytest.mark.parametrize("boundary", [3, 6, 9, 12, 15, 18, 21, 24]) +def test_native_host_death_after_each_record_or_head_keeps_the_epoch_burned(tmp_path, boundary): + root, _ = _inventory(tmp_path) + code = """ +import os, sys +from pathlib import Path +from auto_coder.management.recovery_epoch_state import _hold_recovery_epoch +from tests.test_management_recovery_epoch_state import _binding, _complete +original = os.fsync +count = 0 +def barrier(fd): + global count + original(fd) + count += 1 + if count == int(sys.argv[2]): + print('DURABLE_BOUNDARY:' + str(count), flush=True) + sys.stdin.readline() +with _hold_recovery_epoch(Path(sys.argv[1]), expected_uid=os.geteuid()) as state: + os.fsync = barrier + _complete(state, _binding(state.inventory)) +""" + process = subprocess.Popen([sys.executable, "-c", code, str(root), str(boundary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert _line(process) == "DURABLE_BOUNDARY:" + str(boundary) + before = _snapshot(root) + finally: + process.kill() + process.communicate(timeout=10) + assert process.returncode == -9 + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("host death produced a reusable recovery grant") + assert _snapshot(root) == before + + +def test_production_factory_ignores_environment_roots_and_has_no_override(tmp_path, monkeypatch): + assert list(inspect.signature(epoch.hold_recovery_epoch).parameters) == [] + seen = [] + + def stop_before_open(root, *, uid): + seen.append((root, uid)) + raise epoch.EpochUnavailable("fixed-root unit observation") + + monkeypatch.setenv("RECOVERY_EPOCH_ROOT", str(tmp_path)) + monkeypatch.setenv("AUTO_CODER_RECOVERY_EPOCH_ROOT", str(tmp_path)) + monkeypatch.setattr(epoch.os, "geteuid", lambda: 0) + monkeypatch.setattr(epoch, "_approved_directory", stop_before_open) + with pytest.raises(epoch.EpochUnavailable, match="fixed-root unit observation"): + with epoch.hold_recovery_epoch(): + pytest.fail("unit seam should stop before real fixed-root access") + assert seen == [(Path("/var/lib/auto-coder/recovery-epoch"), 0)] From faab92049b630f519312ecdee92c2f1417635de6 Mon Sep 17 00:00:00 2001 From: kitamura-tetsuo Date: Fri, 13 Nov 2026 00:19:50 +0900 Subject: [PATCH 02/24] fix(management): poison epoch after conflicting reserved binding --- .../management-recovery-epoch-design.md | 5 ++++- .../management/recovery_epoch_state.py | 9 ++++++++- tests/test_management_recovery_epoch_state.py | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/client-features/management-recovery-epoch-design.md b/docs/client-features/management-recovery-epoch-design.md index 26ac30d9b..b49bd9052 100644 --- a/docs/client-features/management-recovery-epoch-design.md +++ b/docs/client-features/management-recovery-epoch-design.md @@ -271,7 +271,10 @@ An independently opened exclusive flock serializes the whole state invocation. The component rechecks the current process's actual kernel lock, protected inventory identity and bytes, every retained record and the complete namespace before each transition. A forked or released holder, replaced lock/root/record, -changed inventory, conflicting alias or existing phase refuses. +changed inventory, conflicting alias or existing phase refuses. Once a reservation +is durable, a conflicting authorization, evidence or issuer-lifetime binding +poisons that same invocation, even when its caller catches the refusal. A +subsequent call using the original binding cannot continue consumption. | Transition | Persisted fact | Reentry behavior | | --- | --- | --- | diff --git a/src/auto_coder/management/recovery_epoch_state.py b/src/auto_coder/management/recovery_epoch_state.py index 565d7cb09..7a39317b2 100644 --- a/src/auto_coder/management/recovery_epoch_state.py +++ b/src/auto_coder/management/recovery_epoch_state.py @@ -211,7 +211,14 @@ def _create(self, name: str, encoded: bytes) -> None: def _append(self, phase: str, binding: EpochBinding) -> str: self._check() - self._require_binding(binding) + try: + self._require_binding(binding) + except BaseException: + # An already reserved epoch cannot recover from conflicting authority + # in the same issuer lifetime, even when the caller catches refusal. + if self._phase: + self._failed = True + raise if self._phase >= len(_PHASES) or _PHASES[self._phase] != phase: raise EpochUnavailable("recovery epoch transition is out of order or already consumed") previous = _digest(self._records[_PHASES[self._phase - 1] + "-head.json"][1]) if self._phase else "" diff --git a/tests/test_management_recovery_epoch_state.py b/tests/test_management_recovery_epoch_state.py index 96af82a7a..3511369b7 100644 --- a/tests/test_management_recovery_epoch_state.py +++ b/tests/test_management_recovery_epoch_state.py @@ -129,6 +129,25 @@ def test_binding_differences_refuse_before_persistence(tmp_path, field, value): assert _snapshot(root) == initial +@pytest.mark.parametrize("field,value", [("authorization_id", "reviewed-B"), ("evidence_sha256", "e" * 64), ("host_start_token", "999")]) +def test_conflicting_binding_after_durable_reservation_poisons_invocation(tmp_path, field, value): + root, inventory = _inventory(tmp_path) + binding = _binding(inventory) + with pytest.raises(epoch.EpochUnavailable, match="closed, failed or inherited"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + state.reserve(binding) + with pytest.raises(epoch.EpochUnavailable): + state.record_intent(replace(binding, **{field: value})) + before = _snapshot(root) + with pytest.raises(epoch.EpochUnavailable, match="closed, failed or inherited"): + state.record_intent(binding) + assert _snapshot(root) == before + with pytest.raises(epoch.EpochUnavailable, match="existing, partial or conflicting"): + with epoch._hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("an interrupted authorization was reused") + assert _snapshot(root) == before + + @pytest.mark.parametrize("name", ["inventory.json", "epoch.lock"]) def test_missing_or_aliased_fixed_files_are_never_recreated(tmp_path, name): root, _ = _inventory(tmp_path) From 5accde9fdca9db25bbcc998c6aa6010a7815a482 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 01:01:49 +0900 Subject: [PATCH 03/24] feat(management): compose original provenance and current recovery epoch (#2824) --- .../workflows/management-recovery-epoch.yml | 85 ++ Dockerfile | 2 +- .../management-original-attempt-provenance.md | 7 + .../management-original-genesis.md | 176 ++++ ...gement-recovery-containment-observation.md | 205 +++++ .../management-recovery-epoch-design.md | 157 +++- .../management-recovery-epoch.md | 200 +++++ docs/dashboard-observability.md | 53 ++ scripts/prepare_management_test_image.py | 2 +- scripts/recovery_epoch_containment_fixture.py | 189 +++++ scripts/recovery_epoch_fault_tracer.py | 335 ++++++++ scripts/recovery_vm/Dockerfile | 10 + scripts/recovery_vm/compose.yaml | 34 + scripts/recovery_vm/guest.sh | 92 +++ scripts/recovery_vm/prepare.sh | 79 ++ scripts/recovery_vm/runner.sh | 46 ++ ...test_management_containment_closed_host.py | 28 +- .../test_management_recovery_epoch_host.py | 749 ++++++++++++++++++ .../management/bootstrap_supervisor.py | 70 +- .../management/containment_observation.py | 45 +- .../management/containment_route_manifest.py | 77 +- src/auto_coder/management/exclusion_host.py | 25 + .../management/fresh_host_observation.py | 33 +- .../management/original_attempt_evidence.py | 213 ++++- .../management/original_attempts.py | 36 +- src/auto_coder/management/original_genesis.py | 592 ++++++++++++++ .../management/recovery_containment.py | 562 +++++++++++++ .../management/recovery_containment_model.py | 309 ++++++++ .../recovery_containment_profile.py | 306 +++++++ .../recovery_containment_resources.py | 136 ++++ src/auto_coder/management/recovery_epoch.py | 377 +++++++++ .../recovery_epoch_authorization.py | 236 ++++++ .../management/recovery_epoch_bootstrap.py | 83 ++ .../management/recovery_epoch_consumer.py | 414 ++++++++++ .../management/recovery_epoch_peer.py | 415 ++++++++++ .../management/recovery_epoch_resources.py | 132 +++ .../management/recovery_epoch_state.py | 27 +- .../management/recovery_source_profile.py | 78 ++ tests/test_build_provenance.py | 2 +- ...t_management_archived_cutover_container.py | 79 +- ...st_management_cutover_fixture_contracts.py | 1 + tests/test_management_original_genesis.py | 432 ++++++++++ ...est_management_original_genesis_capture.py | 72 ++ tests/test_management_original_genesis_cli.py | 36 + tests/test_management_recovery_containment.py | 429 ++++++++++ ...nagement_recovery_containment_resources.py | 231 ++++++ ...management_recovery_epoch_authorization.py | 212 +++++ ...test_management_recovery_epoch_consumer.py | 390 +++++++++ ..._management_recovery_epoch_fault_tracer.py | 135 ++++ tests/test_management_recovery_epoch_host.py | 223 ++++++ tests/test_management_recovery_epoch_join.py | 426 ++++++++++ ...management_recovery_epoch_observability.py | 71 ++ tests/test_management_recovery_epoch_peer.py | 368 +++++++++ 53 files changed, 9611 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/management-recovery-epoch.yml create mode 100644 docs/client-features/management-original-genesis.md create mode 100644 docs/client-features/management-recovery-containment-observation.md create mode 100644 docs/client-features/management-recovery-epoch.md create mode 100644 scripts/recovery_epoch_containment_fixture.py create mode 100644 scripts/recovery_epoch_fault_tracer.py create mode 100644 scripts/recovery_vm/Dockerfile create mode 100644 scripts/recovery_vm/compose.yaml create mode 100644 scripts/recovery_vm/guest.sh create mode 100644 scripts/recovery_vm/prepare.sh create mode 100644 scripts/recovery_vm/runner.sh create mode 100644 scripts/test_management_recovery_epoch_host.py create mode 100644 src/auto_coder/management/original_genesis.py create mode 100644 src/auto_coder/management/recovery_containment.py create mode 100644 src/auto_coder/management/recovery_containment_model.py create mode 100644 src/auto_coder/management/recovery_containment_profile.py create mode 100644 src/auto_coder/management/recovery_containment_resources.py create mode 100644 src/auto_coder/management/recovery_epoch.py create mode 100644 src/auto_coder/management/recovery_epoch_authorization.py create mode 100644 src/auto_coder/management/recovery_epoch_bootstrap.py create mode 100644 src/auto_coder/management/recovery_epoch_consumer.py create mode 100644 src/auto_coder/management/recovery_epoch_peer.py create mode 100644 src/auto_coder/management/recovery_epoch_resources.py create mode 100644 src/auto_coder/management/recovery_source_profile.py create mode 100644 tests/test_management_original_genesis.py create mode 100644 tests/test_management_original_genesis_capture.py create mode 100644 tests/test_management_original_genesis_cli.py create mode 100644 tests/test_management_recovery_containment.py create mode 100644 tests/test_management_recovery_containment_resources.py create mode 100644 tests/test_management_recovery_epoch_authorization.py create mode 100644 tests/test_management_recovery_epoch_consumer.py create mode 100644 tests/test_management_recovery_epoch_fault_tracer.py create mode 100644 tests/test_management_recovery_epoch_host.py create mode 100644 tests/test_management_recovery_epoch_join.py create mode 100644 tests/test_management_recovery_epoch_observability.py create mode 100644 tests/test_management_recovery_epoch_peer.py diff --git a/.github/workflows/management-recovery-epoch.yml b/.github/workflows/management-recovery-epoch.yml new file mode 100644 index 000000000..01e15ec41 --- /dev/null +++ b/.github/workflows/management-recovery-epoch.yml @@ -0,0 +1,85 @@ +name: Management Recovery Epoch Acceptance + +'on': + pull_request: + +permissions: + contents: read + actions: read + +concurrency: + group: ${{ github.workflow }}-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + recovery-epoch: + name: Recovery epoch (${{ matrix.scenario }}) in isolated VM + runs-on: ubuntu-24.04 + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + scenario: [grant, kill-after-intent, kill-after-consume, lost-reply] + env: + UV_LOCKED: "1" + PYTHONDONTWRITEBYTECODE: "1" + RECOVERY_FIXTURE: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }} + APPROVED_AUTHORIZATION_ID: ci-recovery-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }} + RECOVERY_SCENARIO: ${{ matrix.scenario }} + steps: + - name: Checkout exact reviewed source + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.13.6" + - name: Install locked test dependencies + run: uv sync --locked --dev --extra test + - name: Require actual hardware virtualization + run: | + set -euo pipefail + test -c /dev/kvm + test -r /dev/kvm + test -w /dev/kvm + - name: Build and verify exact installed management image + run: uv run python scripts/prepare_management_test_image.py + - name: Prepare the approved fresh recovery VM + env: + FIXTURE_READ_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + umask 077 + token_file="$RUNNER_TEMP/recovery-source-read-token" + trap 'rm -f "$token_file"' EXIT + printf '%s' "$FIXTURE_READ_TOKEN" > "$token_file" + docker pull caddy:2 + manager_id="$(docker image inspect --format '{{.Id}}' "$AUTOCODER_FRESH_TEST_BASE_IMAGE")" + caddy_id="$(docker image inspect --format '{{.Id}}' caddy:2)" + bash scripts/recovery_vm/prepare.sh "$RECOVERY_FIXTURE" "$APPROVED_AUTHORIZATION_ID" "$manager_id" "$caddy_id" "$token_file" "$RECOVERY_SCENARIO" + - name: Build bounded QEMU runner + run: docker compose --env-file "$RECOVERY_FIXTURE/compose.env" -f scripts/recovery_vm/compose.yaml build + - name: Run actual original producers, containment and one-shot issuer + timeout-minutes: 125 + run: | + set -euo pipefail + docker compose --env-file "$RECOVERY_FIXTURE/compose.env" -f scripts/recovery_vm/compose.yaml run --rm fixture + - name: Remove the ephemeral read credential + if: always() + run: | + rm -f "$RUNNER_TEMP/recovery-source-read-token" + rm -f "$RECOVERY_FIXTURE/input/source-token" + - name: Retain bounded native acceptance diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: management-recovery-epoch-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }} + path: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }}/output/ + if-no-files-found: error + retention-days: 7 diff --git a/Dockerfile b/Dockerfile index d44114f88..0380736b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ RUN groupadd --gid 65532 auto-coder-worker && useradd --uid 65532 --gid 65532 -- COPY --from=python-environment /opt/auto-coder/.venv /opt/auto-coder/.venv ENV PATH="/opt/auto-coder/.venv/bin:/home/node/.opencode/bin:${PATH}" # Embed the trusted build input so the installed artifact reports it without Docker/Git/network access. -RUN python -m auto_coder.build_provenance embed +RUN python -B -m auto_coder.build_provenance embed COPY --from=build /usr/local/bin/opencode /usr/local/bin/opencode COPY --from=build /usr/local/bin/codex /usr/local/bin/codex WORKDIR /workspace diff --git a/docs/client-features/management-original-attempt-provenance.md b/docs/client-features/management-original-attempt-provenance.md index d47c1b87c..18b561c91 100644 --- a/docs/client-features/management-original-attempt-provenance.md +++ b/docs/client-features/management-original-attempt-provenance.md @@ -23,6 +23,13 @@ its current fences; it must separately decide whether that explicitly bounded coverage supplies the history its own contract requires. Neither enrollment nor a matching evidence object supplies recovery or launch authority. +The separate `ordinary-original-genesis-v1` profile described in +[Original ordinary deployment genesis and retained lineage](management-original-genesis.md) +adds an affirmative original root-creation and ordinary first-launch baseline. +It uses `ordinary-original-history-v1` authority and the independent +`derive_lineage`/`verify_lineage` API. It does not change this prospective-only +profile's pre-enrollment gap or convert any earlier deployment. + ## Original producer and lifetime An operator must separately provision a new canonical mode-0700 history directory diff --git a/docs/client-features/management-original-genesis.md b/docs/client-features/management-original-genesis.md new file mode 100644 index 000000000..fd2b30fbf --- /dev/null +++ b/docs/client-features/management-original-genesis.md @@ -0,0 +1,176 @@ +# Original ordinary deployment genesis and retained lineage + +The explicitly selected `ordinary-original-genesis-v1` producer supplies the +prospective original baseline required by the recovery epoch's lineage reader. +It implements the original-evidence responsibility in #2839. It creates no +recovery epoch, issues no recovery grant, and does not authorize a deployment. +The existing ordinary bootstrap is still the separately authorized process +that launches its configured management child. + +An original deployment cannot be inferred from an empty old directory or a +currently stopped Docker object. This profile therefore has two distinct +original capture boundaries before the ordinary bootstrap can run: + +1. `prepare_original_genesis_roots` consumes a protected explicit new-deployment + authorization. It first creates an exclusive original claim, then creates + the state, management-operation journal, OAuth and history directories with + `mkdir` and no `exist_ok`. Every preexisting path is refused. The original + root receipt binds the actual directory inodes, the selected protected + configuration files, producer source and Linux process lifetime. A partial + claim remains unavailable and is never repaired or retried. +2. `capture_original_genesis` consumes the separate final authorization naming + the exact manager and paired proxy Docker IDs. It requires both objects to + be `created`, with zero PID, zero restarts and never-started timestamps. It + compares the complete local-engine inventory, effective original commands, + protected configuration mounts, original root identities, and the complete + installed Python package with the executing reviewed producer source. The + selected manager and proxy may not have other discovered management routes. + This capture reads Docker only; it does not create, start, stop or reconfigure + Docker objects. + +These observations do not convert a legacy baseline into original evidence. +The root creation receipt must already exist, and its authorization and +configuration identities must match the final selection. Existing or partial +history, a reused authorization claim, replaced roots, changed source, +conflicting original files or a previously started object remain unavailable. + +## Explicit source and configuration selection + +`BootstrapConfig.original_genesis` is optional. Selecting this producer requires +its exact value `/bootstrap/original-genesis.json`, the ordinary bootstrap +command and the fixed installed management console entrypoint. It cannot be +combined with fresh initialization, NEW initialization, inherited exclusions +or the installed fresh-runtime selector. Omitting it preserves existing +ordinary behavior. NEW/#2672 and ordinary fresh/upgrade refusal behavior are +unchanged. + +The selected ordinary manager has these original mounts: + +| Role | Container path | Access | +| --- | --- | --- | +| Original bootstrap configuration and genesis files | `/bootstrap` | Read-only | +| Original child, handoff and recovery stores | `/management-child-state` | Read/write | +| Original management-operation journal | `/management-operations` | Read/write | +| Originally created OAuth root | `/management-oauth` | Read/write | +| Original history authority and lock | `/management-original-history` | Read-only | + +The same mapped inodes are independently observed by the host and bootstrap. +The fixed configuration members are `/bootstrap/config.json` and +`/bootstrap/management.yaml`. The paired proxy has the ordinary Caddy command, +its exact protected configuration bind mount, and the manager's network +namespace. All original files are owner-only regular files with one link; +protected original directories have mode `0700`. + +Root authorization JSON contains exactly the fields of +`GenesisRootAuthorization`: `profile` (`ordinary-original-roots-v1`), `purpose` +(`separately-authorized-new-deployment`), `deployment_id`, canonical UUID +`authorization_id`, `bootstrap_path`, `management_path`, `proxy_config_path`, +`state_root`, `journal_root`, `oauth_root`, and `history_root`. The four root +paths must not yet exist, and their parent directories must already be +protected. Configurations and authorization files must already be protected. +The writer records `original-genesis-root-claim.json` and +`original-genesis-roots.json` beside the bootstrap configuration. + +The later `GenesisSelection` has `profile` (`ordinary-original-genesis-v1`), +`purpose`, `deployment_id`, `authorization_id`, `management_id`, `proxy_id`, +`history_root`, `bootstrap_path`, `management_path`, `proxy_config_path`, and +`root_authority_path`. Its deployment, authorization, configuration and history +scope must match the actual original root authority. Docker IDs are exact +64-character IDs, not names or mutable tags. Both inputs use canonical JSON. + +The host-root CLI stages are: + +```sh +python -m auto_coder.management.original_genesis \ + --prepare-roots /protected/new-deployment-root-authorization.json \ + --log-file /protected/logs/original-root-capture.log + +# The separately authorized deployment workflow creates the exact ordinary +# manager and proxy with the original roots, then writes the final selection. +python -m auto_coder.management.original_genesis \ + --authorization /protected/new-deployment-final-authorization.json \ + --log-file /protected/logs/original-genesis-capture.log +``` + +This module does not perform the deployment workflow represented by the comment. +Loguru writes producer diagnostics to the selected file and stdout. Log files +must be outside the pristine root namespaces during host capture. + +## Native bootstrap boundaries + +Before any child or management effect, the actual `_run_owned` ordinary path +opens its original owner file independently and obtains its exclusive kernel +flock. Its first owner uses exclusive file creation; an interrupted first +owner cannot be reclassified as an untouched root. The bootstrap obtains a +shared history flock through the original read-only mount, verifies the exact +original authority and genesis, checks current source/configuration/root +identities, and requires the original history to remain empty for its first +native initialization. + +The existing native initializers create all four original SQLite stores under +an owner-only umask. The bootstrap durably records their exact inodes in +`original-genesis-stores.json`, followed by +`original-genesis-first-launch.json`, which binds the exact first operation, +attempt, producer lifetime and native receipt. The normal ordinary launch path +uses those same operation and attempt IDs. These records describe original +boundaries; neither record asserts that a child actually launched or that the +application became ready. + +An interruption before a required native record remains unavailable. The +bootstrap cannot recreate missing originals or replay the first owner. An +originally recorded but unresolved first launch retains its unknown outcome. +Normal store and unknown-state refusal rules continue to apply on later +bootstrap reconstruction. + +## Full retained lineage API + +`original_attempts.read_frontier` recognizes `ordinary-original-history-v1` +authority in addition to the existing prospective fresh-attempt authority. +The new authority pins the exact original genesis file, which fixes the +baseline instance and Docker object before the first successor exists. +Every supported later ordinary fresh, upgrade and exclusion-cutover host +preflight must preserve that same history binding. A successor cannot select +an arbitrary first predecessor or reuse a baseline protected root. All +original failed branches remain in the append-only frontier. + +`original_attempt_evidence.derive_lineage(binding, history_fd, owner_fences)` +and `verify_lineage(submitted, binding, history_fd, owner_fences)` independently +read original genesis, every retained frontier event and every expected native +store. The caller holds its own current history shared flock and exactly one +current shared owner flock for each native generation. A never-created branch +has its affirmative original terminal boundary and no invented owner file. +The reader joins each native successor to its predecessor's exact owner inode +and the original retained OAuth root. + +The frozen `OriginalLineageEvidence` exposes original `genesis`, `attempts`, +`generations`, `originals`, `frontier`, and `unavailable` reasons. Each +`OriginalGeneration` includes the exact management/source/configuration/root +scope, native owner identity, expected stores and factual `outcome`. The +original baseline also pins its proxy identity. A fresh successor's proxy is +not invented from an intent that never recorded one; current containment must +independently establish that pair. + +Complete original evidence with an `unknown` or unresolved application outcome +is distinct from missing original evidence. For example, the native store +receipt is produced after the original first-launch claim, so a missing claim +is a provenance gap even if a status reader can report `unknown`. The lineage +reader preserves the unknown and every retained native record. It never +checkpoints SQLite, clears unknowns, repairs evidence, rewrites exclusions, or +issues launch authority. A nonempty unresolved WAL or rollback journal remains +unavailable. Existing `derive`/`verify` prospective-reader behavior is preserved. + +## Verification boundaries + +`tests/test_management_original_genesis.py` exercises actual protected +filesystem publication, affirmative root creation, original claim refusal, +root/source/configuration pinning, actual native store initialization under an +exclusive owner flock, shared-fence independent derivation, preserved unknown +outcomes, complete failed branches, and missing-original refusal. Docker +observations and container bind mapping are explicit component substitutes; +these tests are not real-Docker, first-child or recovery-epoch acceptance. + +The production interfaces are suitable for isolated real-Docker and host-root +acceptance. Full #2824 acceptance additionally requires the actual complete +current containment verifier, issuer, OS descriptor transfer, consumer receipt +and durable epoch boundaries. Component provenance tests do not substitute for +that integrated acceptance. diff --git a/docs/client-features/management-recovery-containment-observation.md b/docs/client-features/management-recovery-containment-observation.md new file mode 100644 index 000000000..ee478843b --- /dev/null +++ b/docs/client-features/management-recovery-containment-observation.md @@ -0,0 +1,205 @@ +# Current containment of an ordinary recovery scope + +Issue #2840 adds a read-only `ordinary-recovery-v1` observation profile for a +separately approved set of original ordinary management generations and one +exact prelaunch recovery consumer. It supplies current facts to the recovery +issuer. It does not establish original-history completeness, repair an original, +issue a grant, create operation admission, start a child or assert readiness. +The original `ordinary-v1`, `quarantined-original-v1`, fresh replacement and NEW +admission profiles keep their existing contracts. + +## Fixed authority and finite selection + +The production reader loads only +`/var/lib/auto-coder/approved-recovery-containment-enrollments/.json`. +The directory must be canonical, root-owned and mode 0700; the file must be a +canonical, root-owned, single-link mode-0600 regular file. No request or environment +variable can substitute the enrollment root. A separate trusted operator installs +approval; the observer cannot create, repair or replace it. + +The exact schema is `approved-management-recovery-containment-v1`. Its fields are +`deployment_id`, `enrollment_id`, `authorization_id`, `profile`, `engine_id`, +`source_identity`, `originals`, `target`, `watchers`, and +`approved_observation_only: true`, together with `schema`. Unknown fields, aliases, +unsorted membership and incompatible versions refuse. The typed definitions and +strict parsers are in `recovery_containment_model.py`. + +| Selection | Independently bound information | +| --- | --- | +| Original generation | Generation and instance IDs; exact manager/proxy IDs and images; bootstrap, management and Caddy configuration file identities; complete actual bind roots; current-state selection | +| Started original | Original boot ID; protected original manager/proxy lifetime capture; exact original owner file; live original namespace holder and its actual retained descriptor | +| Never-started original | Existing Docker objects currently in the pristine `created` state; no invented owner, process lifetime, prior record or namespace pin | +| Target | Exact separate manager/image/instance; protected bootstrap and management configuration; complete bind roots; prepared owner inode; distinct loopback published port | +| Watcher | Exact installed systemd unit, actual installed module and protected configuration, and the existing `proxy-watch.lock` that the real watcher takes before tick/reconcile | + +There are at most 16 original pairs and 32 watchers. Their complete eventual peer +custody set must also satisfy `3 + 2 × started originals + watchers <= 64`; the +three fixed resources are history, target owner and deployment epoch. Original +pairs, target, owner files and controller files cannot alias. Reusing one proxy +object across multiple enrolled generations is outside this finite profile. + +A lineage branch with positive original never-created evidence and no current +Docker object is not fabricated as a containment pair. The issuer joins that +original history to absence from the independently enumerated whole engine. Any +actual branch object remains subject to complete selection and profile checks. +Current `created` state alone supplies no historical never-created certificate. + +## Actual executable source and configuration + +The observer copies the complete installed `auto_coder` package from the exact +selected manager objects on the local system Docker engine. It reads a bounded +uncompressed archive without executing it. Every Python source filename and byte +sequence contributes to the same length-framed identity as +`original_attempts.source_identity()`. The protected enrollment, current installed +observer and every selected manager must agree on that identity. + +Absolute or ambiguous archive paths, duplicate members, links, special files, +missing original-history modules, bytecode and native import extensions refuse. +Package `.pyc`, `.pyo`, `.so` or `.pyd` files cannot substitute different executable +bytes for the observed Python source. The supported container environment sets +`PYTHONDONTWRITEBYTECODE=1` before the first bootstrap import. Existing cached +package bytecode remains unsupported; setting the environment later cannot make +it acceptable. The production Dockerfile embeds build provenance with `python -B` +to avoid introducing package bytecode during that build step. + +Effective commands must be the actual ordinary bootstrap entrypoint, the ordinary +fresh entrypoint for an original successor, or the target's explicit +`--recovery-epoch` entrypoint. Originals using fresh initialization keep their +unknown or failed historical outcome; this observation path never requires or +invents a successful fresh outcome. Their historical justification belongs to the +independent lineage producer and issuer join. + +The mount reader independently checks the full current Docker inspection, +protected source/configuration bytes, actual bind root device/inode/mode, Python +resolution, runtime command and effective configuration mapping. Interpreter, +package and working-directory shadowing refuse. Original genesis uses the fixed +read-only `/management-original-history` mapping and +`/bootstrap/original-genesis.json` input. The target uses the fixed +`/bootstrap/recovery-epoch.json` input and a separately checked installed runtime; +its version root is read-only at the same absolute host/container path. The issuer +owns the additional tested-source and READY verification. + +The target's state and journal roots are separate from every original mutable +root. Its OAuth root remains the exact original OAuth path and object. The new +published endpoint must differ from every original published port, so a target +listener cannot hide an old listener. The actual target listener must be the +local Docker userland proxy for its exact container address and endpoint. + +The target's read-only custody mappings are `/management-recovery` from the fixed +host `/var/lib/auto-coder/recovery-epoch-peer`, the exact history lock at +`/management-recovery-history/history.lock`, and the independently selected old +owner files at `/management-recovery-originals//owner.lock`. Indices follow +the canonical ordered original list. Their actual device/inode and configured +roles must agree. These inputs do not themselves establish acquired custody. + +## Complete current controllers and kernel facts + +Whole-engine enumeration is unconditional. Every selected object must exist on +the exact local engine. Unselected objects sharing selected images, management +commands, selected namespaces or original bind sources remain management +candidates and refuse. Docker endpoint overrides, foreign engines, changing +inventories and unsupported profiles refuse. Ordinary unrelated objects remain +visible in the complete engine evidence. + +The sibling `.routes.json` must use +`approved-management-recovery-routes-v1`, bind the exact enrollment hash, and carry +separate explicit complete-route approval. It reuses the existing exhaustive +system/user systemd, activation, cron/at and installed-script readers over the +union of all original and target IDs and actual roots. Every restarter must be +technically inhibited; restart policies must be `no`. Unknown exclusions and +unsupported remote origins cannot be converted into closure by an operator list. +Each real watcher must use its enrolled source/configuration and actual existing +controller lock. The observer changes no unit, restart policy, file or controller. + +For each started original, the observer checks the exact captured manager/proxy +lifetimes and mount identities, every remaining cgroup task, absence of captured +escaped tasks, the whole host's task references to the retained original network +namespace, and absence of the old host listener. Stopped Docker state or an old +namespace number is insufficient. The original boot must still match, and a live +holder must still retain the genuine original namespace object. + +`RecoveryResources` names a current host holder by boot, PID and process start +token. Its `original_owner_fds`, `controller_fds` and `namespace_fds` name exact +existing descriptors. Original names use the enrolled `generation_id`; controller +names use the exact watcher unit. The observer independently reopens procfs +references, holds observation references, checks protected paths and file objects, +reads kernel flock records, and pins holder lifetimes while it samples. + +Every original-owner flock must be SH and independently attributed to that issuer +PID. An inherited parent's SH descriptor is refused. Every real watcher lock must +be held EX by that same issuer. The target separately holds its own owner EX under +its actual prelaunch process. File replacement, released locks, reused descriptor +numbers, holder exit and missing resource membership refuse. No observation claims +that these present fences existed during the historical gap. + +## Library and CLI + +The production library exposes: + +- `load_enrollment(enrollment_id) -> RecoveryEnrollmentObservation` +- `observe(enrollment_id, resources) -> RecoveryContainmentReport` +- `verify(enrollment_id, previous, resources) -> RecoveryContainmentReport` + +Missing or unsupported facts raise `RecoveryContainmentUnavailable`. A positive +report has status `observed-current-recovery-containment`, typed complete facts, +a fresh observation ID and boot/process sampling interval, and a deterministic +`evidence_sha256` over stable scope/source/resource/controller/kernel facts. +Diagnostic observer PID and sampling timestamps are outside that stable digest. +`grant_issued`, `history_asserted` and `admission_created` are always false. + +```sh +python -m auto_coder.management.recovery_containment observe \ + --enrollment separately-approved-recovery \ + --resources /private/current-issuer-descriptors.json +python -m auto_coder.management.recovery_containment verify \ + --enrollment separately-approved-recovery \ + --resources /private/current-issuer-descriptors.json \ + --previous /private/previous-current-observation.json +``` + +CLI success exits 0. Unavailable evidence exits 2 with a structured unavailable +result. A resource input only identifies existing descriptors; supplying a number +cannot create a lock, pin or approval. Reports expose hashes and identities rather +than configuration contents. Logger diagnostics may also appear on stdout, so a +machine caller selects the structured JSON result line. + +The separate verifier repeats real source, kernel, controller and holder reads. +It rejects changed facts, malformed or authorizing reports, invalid digests, +foreign boots, future samples and samples older than ten minutes. The earlier +read-only sampling command may have exited; its PID is attribution. Actual issuer +and namespace-resource producers must remain alive and independently inspectable. +A saved positive result never replaces a current check or establishes continuity. + +## Reproduction and observability + +Component tests use the required repository runner: + +```sh +CI=true UV_NO_SYNC=1 PYTHONPATH=.:src bash scripts/test.sh --ci-tests-only \ + tests/test_management_recovery_containment.py \ + tests/test_management_recovery_containment_resources.py +``` + +The parser/source/profile tests are component evidence. The resource tests use real +independent processes, SH/EX flocks, namespace descriptors, process death and file +replacement. Neither suite by itself proves whole-host route or Docker closure. + +The native recovery harness uses the existing separately provisioned disposable +KVM/QEMU VM authorization, initial host namespaces, PID 1 systemd and the local +system Docker engine. `scripts/recovery_epoch_containment_fixture.py` captures +live original generations before stopping them, retains genuine namespace FDs, +runs and inhibits actual installed watchers, and constructs the separately +approved finite fixture scope. It imports production observation readers and +creates no substitute successful evidence. The production observer does not +import this fixture helper. Native runs must include genuine lineage and failed +branches, the prelaunch target, actual unrelated workers and preserved original +records; a component-only pass cannot be reported as native acceptance. + +Observation logging uses Loguru's configured file and stdout sinks. Its explicit +fields are `observation_only=True`, `recovery_grant=False` and +`admission_created=False`. No operation journal admission, dispatch, recovery +success or readiness event is emitted. The native harness verifies original +journal/state and unrelated worker lifetime/request fingerprints across the +production entrypoints. Diagnostic sampling output is not an operational success +row in the dashboard; the shared dashboard observability documentation records +this distinction. diff --git a/docs/client-features/management-recovery-epoch-design.md b/docs/client-features/management-recovery-epoch-design.md index b49bd9052..3d9c7c5db 100644 --- a/docs/client-features/management-recovery-epoch-design.md +++ b/docs/client-features/management-recovery-epoch-design.md @@ -6,15 +6,18 @@ result is a current, exact-target, one-use grant composed from independently verified original history, current containment and retained fences. This fragment records implementation boundaries found at source [`e742338b3f311e6d30a22a8f5341b8072513d813`](https://github.com/kitamura-tetsuo/auto-coder/commit/e742338b3f311e6d30a22a8f5341b8072513d813). -It is a design and prerequisite audit, not a recovery procedure or evidence -that a production grant can currently be issued. - -The bounded durable-state prototype accompanying this design does not wire a -production issuer or authorize recovery. In particular, an unused state API, -component test, operator inventory or saved prerequisite verdict cannot satisfy -the full Issue. Existing unknown-outcome refusals remain correct. No original -claim, missing journal, operation key, target root or initialization outcome is -repaired or reset by this design. +This is the retained audit of that original source, not a recovery procedure or +live-operation authority. The implemented producer and its current interfaces are +described in [management-recovery-epoch.md](management-recovery-epoch.md), +[management-original-genesis.md](management-original-genesis.md), and +[management-recovery-containment-observation.md](management-recovery-containment-observation.md). +The missing-capability tables below describe the audited base and explain why +the two additional evidence-only prerequisites were necessary. + +An unused state API, component test, operator inventory or saved prerequisite +verdict cannot satisfy the full Issue. Existing unknown-outcome refusals remain +correct. No original claim, missing journal, operation key, target root or +initialization outcome is repaired or reset by this design or implementation. ## Existing authoritative producers @@ -121,9 +124,10 @@ single deployment's ownership or controller scope. ## Minimal prerequisite capabilities proposed for review -These are proposed capability boundaries, not newly published Issues, -authorizations, changes to #2824's Objective, or waivers of its Requirements. -The existing recovery issuer remains responsible for REQ-101 through REQ-108. +These capability boundaries preserve #2824's Objective and Requirements. +The existing recovery issuer remains responsible for REQ-101 through REQ-108; +extracting prerequisites neither supplies live-operation authorization nor +waives its integrated obligations. **Prospective deployment genesis and complete lineage bridge.** Establish an original, protected binding between a never-before-launched deployment origin @@ -154,17 +158,22 @@ or consuming an epoch. Their state/lifetime and runtime authorities differ. The host issuer should compose their defined outputs at one durable issuance boundary rather than silently invent both producers inside a nominal adapter. -**Unresolved supported-profile decision:** choose the production origin and -engine model for the first supported complete recovery deployment. Two concrete -paths are a prospective genesis path using the local system engine, with its -own explicit closed-controller admission, or extending the genesis/lineage and -containment observers to the existing private-engine NEW origin. The former -changes the supported genesis admission surface; the latter must bind both -outer private-engine custody and inner deployment facts. Existing component -code does not select between them. Neither option authorizes migrating, -re-enrolling or launching the current incident's deployment. +**Selected implementation profile:** retain the ordinary deployment on the local +system Docker Engine. Add prospective original provenance at its actual +original production boundary, before any supported first launch or successor +admission. The trusted producer must create and authenticate a genuinely new +deployment identity and its original roots, configuration and history binding; +an empty preexisting directory is not proof that a prior launch never happened. +All ordinary bootstrap routes in this profile must enforce that binding. + +This follows the existing-deployment profile in #2824 and leaves NEW/#2672's +private-engine admission unchanged. Reusing the private-engine NEW receipt was +considered, but would add an unnecessary engine/topology dependency. A missing +producer is implementation work, not by itself an architectural approval +boundary. The selected prospective profile still cannot migrate, re-enroll or +launch the current incident's deployment, or fill any missing original history. -## Requirement-by-requirement implementation audit +## Requirement-by-requirement audit of the original base | Requirement | Reusable implementation | Missing production fact or integration | | --- | --- | --- | @@ -177,12 +186,33 @@ re-enrolling or launching the current incident's deployment. | REQ-107 | Existing old-root unknown-outcome and first-launch refusals. | Explicit new-epoch authorization and original-exclusion composition that cannot reuse an old root, plan, claim or operation key. | | REQ-108 | Real original-host boundary fixtures, Docker fresh tests and disposable closed-host fixture components. | All integrated AS-B01 through AS-B05 scenarios through the recovery production issuer and actual prerequisite verifiers. Existing component receipts do not satisfy this row. | -The durable prototype may verify persistence and locking behavior independently. -It does not make any missing row ready, and the absence of an issuer entry point -must not be presented as successful fail-closed completion of the Issue. +The durable component verifies persistence and locking independently. It cannot +establish any of these prerequisite joins by itself. The current implementation +connects the new original-genesis and full-scope containment producers to the +zero-argument issuer; all integrated acceptance obligations remain required. ## Issuer and consumer protocol boundary +`recovery_epoch_peer.PeerSession` implements bounded authenticated transport and +current custody observation. The issuer supplies an `ExpectedPeer` already +bound by its real Docker/target validator, the complete `HeldFence` set and any +`HeldResource` network-namespace pins. The session pins the peer with a pidfd, +checks `SO_PEERCRED` and the exact current start-token/boot lifetime, exchanges a +fresh challenge-bound offer and receipt, and returns immutable `PeerCustody`. +`session.check(custody)` repeats actual file, lock, namespace and peer-lifetime +observations. There is no grant method: the issuer separately owns durable +consumption and grant ordering. Caller-owned fences remain open on refusal; +the session closes its own socket/PID pin and any unexpected received FDs. + +Real subprocess tests in `tests/test_management_recovery_epoch_peer.py` exercise +independent SH/target EX acquisition, passed EX open-file-description custody, +network namespace transfer, observable queued/received/acknowledged boundaries, +malformed or incomplete receipts, inherited-as-independent locks and current +custody loss. The independent consumer creates a local socket pair and passes +the issuer endpoint over an inherited mailbox, preserving real consumer PID +credentials without requiring a filesystem listener. These are kernel/peer +component tests, not Docker, complete deployment scope or recovery-grant tests. + A prospective issuer needs a fresh authenticated local challenge bound to the fixed deployment, explicit authorization, exact target/preparation and exact consumer process/boot lifetime. The existing `SO_PEERCRED` and container/ledger @@ -248,7 +278,7 @@ receipts must be collected and required by an appropriate isolated profile; no missing-node or skip allowance may silently convert unavailable authority into acceptance. -## Durable state component in this draft +## Durable state component and its authority boundary `recovery_epoch_state.hold_recovery_epoch()` has no arguments. Its production root is fixed at `/var/lib/auto-coder/recovery-epoch`; environment variables, @@ -256,7 +286,7 @@ request paths, target names and preparation names cannot select another root. The factory requires the approved Unix owner and protected, canonical directory ancestors. It does not provision or repair a missing directory, inventory or lock. An operator would have to separately review and install the inventory; -this draft performs no such operation. +the feature implementation performs no such live operation. The finite `EpochInventory` binds the exact root and lock device/inode identities, deployment, authorization, sorted unique enrollment set, request, preparation, @@ -264,8 +294,8 @@ Docker target, source digest and configuration digest. An `EpochBinding` must match those fields and additionally binds the verified-evidence digest and the actual current issuer boot/PID/start lifetime. The component validates those joins and their exact schema, but cannot establish that the supplied evidence -was independently produced. The missing issuer must establish that fact before -calling it; inventing a digest is not an authorization mechanism. +was independently produced. The issuer in `recovery_epoch.py` establishes that +fact before calling it; inventing a digest is not an authorization mechanism. An independently opened exclusive flock serializes the whole state invocation. The component rechecks the current process's actual kernel lock, protected @@ -307,21 +337,64 @@ bash scripts/test.sh --ci-tests-only tests/test_management_recovery_epoch_state. The component uses the existing Loguru configuration with console and rotating file sinks. Its phase records explicitly carry `grant_delivered=False`; they -never report authenticated readiness or recovery success. The eventual issuer's -production trace integration remains part of the unfinished contract below. +never report authenticated readiness or recovery success. The issuer's real +phase, refusal and dispatch events are documented in +`docs/dashboard-observability.md` and retain the same distinction. + +## Prelaunch consumer custody + +`recovery_epoch_consumer.connect_recovery_epoch(config_path, target_owner_fd, +timeout=...)` accepts only `/bootstrap/recovery-epoch.json` and connects only +`/management-recovery/issuer.sock`. The target's independently verified Docker +configuration must bind that configuration's bytes, the read-only mounted issuer +directory and its device/inode identity. The configuration contains the exact +request, authorization and target IDs; each independently acquired original or +history shared fence's local path and device/inode; the target owner identity; +and every transferred controller, deployment or network-namespace identity. +Host paths are never assumed to be valid target paths. + +The consumer authenticates the connected issuer with `SO_PEERCRED` and obtains +its lifetime pin directly from the kernel with `SO_PEERPIDFD`. That capability is +mandatory: a private Docker PID namespace may expose the host peer as PID zero, +so a caller-supplied PID or pidfd cannot substitute for socket-bound kernel +identity. The consumer independently acquires the declared shared locks and +checks its already held target exclusive lock. It acknowledges only the exact +challenge-bound received resource set, with current kernel flock records for +exclusive descriptors and exact nsfs objects for namespace pins. It accepts a +grant only after that receipt and only with the exact request/challenge binding, +all four durable digest fields and `granted="recovery-epoch-consumed"`. + +`RecoveryEpochConsumer` retains its original shared locks, transferred exclusive +locks and namespace pins, socket and issuer pidfd. The caller continues to own +the target exclusive descriptor. `check_current()` rechecks custody and the +issuer's process and socket lifetime; failure does not implicitly close retained +resources. An explicit `close()` or context exit releases them only when the +owning Supervisor establishes that it is safe. The helper has no child claim, +store creation, launch or readiness operation; those remain the separate child +contract. The configurable overall offer/grant deadline is positive and at most +one hour, allowing full host observations while keeping each protocol send +bounded to at most sixty seconds. + +`tests/test_management_recovery_epoch_consumer.py` uses independent real issuer +and consumer processes, actual flocks, namespace descriptors and kernel peer +pidfds. It observes the offer/receipt/grant barriers, rejects incomplete custody +and stale grants, and proves that both the independent shared fence and received +exclusive fence still prevent a new exclusive contender after issuer death, +until explicit consumer close. Its wire issuer supplies protocol grant bytes; +these tests do not establish Docker, lineage, durable consumption or complete +production grant authority. A real connected production issuer/consumer path +in the isolated native profile is still required for that claim. ## Observability and completion boundary -The unwired persistence/design work adds no production processing origin, -admission grant, success event, worker action or dashboard-owned state. -Consequently it supplies no recovery-success trace and no dashboard claim of -readiness. Wiring the eventual issuer must include its real reservation, -refusal, consumption, dispatch and current-host-loss observations in the -production trace contract and `docs/dashboard-observability.md`, with runnable -tests for those boundaries. +The host issuer adds no GitHub processing attempt, management operation admission, +worker action or dashboard-owned state. Its actual reservation, refusal, +consumption and grant-send observations use the existing Loguru stdout/file +configuration and explicitly distinguish dispatch from current readiness. +`docs/dashboard-observability.md` describes those events and their maintained +checks. Full native VM results must still substantiate the integrated contract. Do not close #2824, merge an implementation as its completion, or deploy a -recovery issuer until the supported-profile decision and missing producers are -resolved, all eight requirements are actually connected, and the integrated -native evidence passes. Completion of a prototype or this audit is not that -result. +recovery issuer until the missing producers are implemented, all eight +requirements are actually connected, and the integrated native evidence +passes. Completion of a prototype or this audit is not that result. diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md new file mode 100644 index 000000000..e2ca81c2c --- /dev/null +++ b/docs/client-features/management-recovery-epoch.md @@ -0,0 +1,200 @@ +# Management recovery ownership epoch + +The host producer in `management/recovery_epoch.py` establishes one new, +explicitly authorized ownership epoch for the finite ordinary local-system-Docker +profile. It joins original provenance from #2839, current full-scope containment +from #2840, quarantined operation inspection from #2823, genuine installed READY, +and actual receiving-process fence custody. Each prerequisite remains evidence +only. None can issue a grant or repair a missing original record. + +The separate #2825 consumer owns a durable child-launch claim, one child launch, +authenticated application readiness, and quarantined admission/dispatch. This +producer's `--recovery-epoch` bootstrap endpoint acquires and retains custody but +never starts a child, creates an operation store, or announces readiness. A +successful socket send is not proof of delivery or an application outcome. + +## Supported original and current scope + +A supported deployment begins with the affirmative original root-creation and +ordinary-bootstrap genesis producers described in +`management-original-genesis.md`. Later supported attempts remain in that same +protected history frontier, including failed branches. Complete original facts +can retain an unknown outcome. A missing expected journal, missing genesis, +unsettled original SQLite sidecar, changed file, or external legacy baseline +remains unavailable. Current enrollment cannot supply historical completeness. + +The current scope is one approved local system Docker engine, exact ordinary +manager/proxy pairs, every actual watcher and normal restart route, and one +separate prelaunch target. Every existing selected original owner inode must be +independently held SH by the issuer and consumer. The issuer also acquires the +actual existing watcher locks EX and its deployment-fixed epoch lock EX. It +transfers those exclusive open file descriptions and the actual retained nsfs +objects, then independently observes their presence in the authenticated +consumer's `/proc/PID/fd` and kernel lock data. Queued SCM_RIGHTS descriptors or +`sendmsg` success do not establish receipt. + +Original lifetime/cgroup/network evidence comes from actual capture while the +original processes were available. A current namespace pin holds the actual +kernel object. A saved historical inode or Docker's stopped status cannot replace +that object, all-task inspection, current namespace occupancy, or endpoint +listener absence. A positively recorded never-created attempt may have no current +Docker object; its exact absence must be present in full-engine observations and +its original failed branch remains retained. Existing objects cannot be omitted +from enrollment to obtain that result. + +The target retains the original OAuth root and inode. Its state and journal roots, +Docker identity, management instance, and published host port are distinct from +every original. The issuer does not stop/start containers, inhibit controllers, +change OAuth, reset old roots, or touch workers. Those conditions must already +hold under separate reviewed operational authorization. Unknown controller routes +or unsupported profiles refuse. NEW/#2672 and the existing ordinary/quarantined +observation profiles retain their admission contracts. + +## Deployment-fixed authority + +The host CLI is `python -m auto_coder.management.recovery_epoch`. It takes no +request, inventory, authorization, or state-root argument. Production files are +fixed; test-only internal factories are not CLI alternatives. + +| Location | Required original authority | +| --- | --- | +| `/var/lib/auto-coder/recovery-epoch/inventory.json` | Canonical version-two `EpochInventory`, including exact directory/epoch-lock identities, deployment, authorization, one enrollment, request, preparation, Docker target, bootstrap/source digests and the full authorization digest. | +| `/var/lib/auto-coder/recovery-epoch/epoch.lock` | Preexisting protected original file; independently acquired EX. The issuer never creates or repairs it. | +| `/etc/auto-coder/recovery-epoch/authorization.json` | Canonical `EpochAuthorization`: exact history binding, protected consumer and READY file identities, configured source repository/read credential/installer/interpreter, exact tested source and socket-directory identity. | +| `/var/lib/auto-coder/approved-recovery-containment-enrollments/.json` and its approved routes file | The complete scope independently checked by the current-containment producer. An operator list is not a containment verdict. | +| `/var/lib/auto-coder/recovery-epoch-peer/issuer.sock` | One protected authenticated local endpoint, created only inside the separately provisioned fixed private directory. Existing endpoints refuse; no stale-socket repair is attempted. | + +Authority directories are root-owned private canonical directories with trusted +ancestors. Files are private original single-link files. The inventory's +`authorization_sha256` binds the complete separately reviewed scope, and that +scope must still match the inventory and original inode/bytes at issuance. +`AUTO_CODER_RECOVERY_EPOCH_ROOT`, `AUTO_CODER_RECOVERY_EPOCH_INVENTORY`, +`AUTO_CODER_RECOVERY_AUTHORIZATION_ROOT`, and +`AUTO_CODER_RECOVERY_CONTAINMENT_ROOT` are refused rather than used as overrides. +No provisioning or alternate-root deduplication path exists in the issuer. + +Version one remains a non-authorizing persistence component format; it cannot +enter the complete issuer. Version two requires canonical retained evidence and +its matching digest. An original request or genesis authorization cannot be +reused as the new recovery request/authorization. + +## Exact installed target and receiving process + +The target selects `BootstrapConfig.recovery_epoch_config` as +`/bootstrap/recovery-epoch.json` and explicitly invokes: + +```sh +python -m auto_coder.management.bootstrap_supervisor \ + --config /bootstrap/config.json --recovery-epoch +``` + +The ordinary supervisor constructor refuses this selected profile, so omitting +the mode or entering the normal first-child path does not bypass recovery. The +endpoint opens the already provisioned target `.supervisor-owner.lock` without +creation, obtains its own independent EX, and requires no previous target launch +or operation state. It connects only to `/management-recovery/issuer.sock` and +holds its own independently opened original/history SH files. + +The endpoint's private configuration binds request/authorization/target, +expected issuer UID, actual socket-directory device/inode, every original +resource name/path/device/inode, target-owner identity, and each transferred +controller/epoch/namespace identity. The issuer independently joins the complete +set to its actual held descriptors. The target mounts the fixed host peer +directory read-only at `/management-recovery`, the exact history lock read-only +at `/management-recovery-history/history.lock`, and selected original owner files +read-only at `/management-recovery-originals//owner.lock`. + +The host authenticates the local peer with SO_PEERCRED, an independently opened +pidfd, the actual Docker leader PID, Linux start token and boot identity. The +consumer uses kernel SO_PEERPIDFD to pin the actual issuer across Docker PID +namespaces; a sender-supplied pidfd or claimed host PID is not authority. Kernels +without this capability refuse. Every resource check remains live after receipt. +The overall consumer verification wait is bounded by the configured bootstrap +verification timeout, up to one hour; individual exchange sends remain bounded. +SIGTERM/SIGINT immediately interrupt a pre-grant wait and close the non-launching +endpoint's resources. Host death does not silently release an already returned +consumer handle's descriptors: its owner must explicitly end custody when safe. + +The selected initial runtime is `management-installed`. The issuer invokes the +actual `ManagementVersionStager.lookup` against original READY, current immutable +tested-source evidence and installed bytes. The management configuration and all +runtime arguments/environment/source identities must agree with +`runtime_for_receipt`. The READY preparation directory must equal the approved +preparation ID. Unsupported executable environment hooks refuse. + +Both the complete interpreter distribution and READY environment must be inside +the exact same-path read-only version-root mount. In particular, a host-only venv +interpreter symlink cannot satisfy target verification. The authorization's +configured interpreter and the runtime's resolved interpreter must be the same +file inside that closure. This permits an actual portable CPython installation +within the separately reserved version root; it does not rewrite a receipt's +original paths or accept an invented READY file. + +## Single-use durability and grant ordering + +The issuer first independently derives and verifies the full original lineage, +inspects all actual quarantined journals, obtains current containment, verifies +exact installed target READY, and observes the consumer's complete acknowledged +custody. It then records the fixed sequence: + +| Phase | Durable consequence | +| --- | --- | +| `reserved` | The one deployment-wide invocation and complete original/current evidence binding are reserved. | +| `intent` | The exact target, authorization, source, full original fingerprints, controller/resource identities and receiving custody remain retained. | +| `consumed` | That epoch can never yield another grant, including after interrupted persistence or a lost reply. | +| `receipt` | A final durable record binds the consumed record and exact evidence; it is not itself delivery proof. | + +Each phase uses O_EXCL publication, file and directory fsync, an immutable +hash-linked head, actual writer-inode readback and complete directory/inventory +revalidation. `evidence_json` retains the canonical verified original fingerprints +and current bindings, and must match `evidence_sha256`. The original outcome is +preserved in that evidence. A conflicting binding after reservation poisons the +current invocation, including when a caller catches the error. Cancellation or +uncertainty during publication also poisons it. Every later invocation rejects +existing or partial phase evidence; nothing resumes, resets or repairs it. + +Original lineage, exact target READY, protected authorization/enrollment, +quarantined journals, actual current containment and receiving custody are all +rechecked before consumption and again after persistence, immediately before the +single send. The grant is bound to the fresh peer challenge and issuer receipt +nonce, exact request/authorization/target, four durable digests and actual issuer +lifetime. There is no retransmission path. A completed send followed by host +loss can race a future consumer's first launch; correctness requires retained +custody and no second grant, not an impossible promise that host death forbids an +already-authorized Popen. + +After the one send, the host remains alive with its fences and session. Loss of +that host, socket or custody makes current consumer checks unavailable. The +historical durable receipt remains unchanged. This endpoint never marks an +application ready; #2825 must own that separate proof and protected operation +lifecycle. + +## Observability and validation + +The CLI and bootstrap endpoint use the existing Loguru stdout plus file sinks. +Durable phase events name `recovery_epoch_phase`, request and target. Explicit +`current_ready=False` accompanies grant-send and unavailable observations. No +GitHub issue/PR processing attempt, dashboard slot, management operation admission +or worker outcome is created by this host ownership protocol. See the matching +entry in `docs/dashboard-observability.md`. + +The maintained component selections cover real persistence failures/process +interruption, exact fixed-scope parsing, independent-process SH/EX/nsfs custody, +actual received descriptors, current peer lifetime and cancellation. Component +fixtures are not original historical authority or live Docker containment. + +```sh +bash scripts/test.sh --ci-tests-only \ + tests/test_management_recovery_epoch_state.py \ + tests/test_management_recovery_epoch_authorization.py \ + tests/test_management_recovery_epoch_peer.py \ + tests/test_management_recovery_epoch_consumer.py +``` + +The separate isolated initial-namespace systemd/Docker VM acceptance must traverse +the actual genesis/attempt producers, complete current observer and exact host +issuer/consumer entrypoints. Its AS-B01 through AS-B05 obligations include +observable contention, missing/mutated/forged prerequisites, all-controller/task/ +listener refusals and interruption/host-death boundaries. Passing the component +selection alone does not establish those native obligations or applicability to +a current incident with missing originals. diff --git a/docs/dashboard-observability.md b/docs/dashboard-observability.md index 073ea211e..af960ef9e 100644 --- a/docs/dashboard-observability.md +++ b/docs/dashboard-observability.md @@ -3611,3 +3611,56 @@ boundary. The installed release acceptance case is listed separately in the required isolated-container profile. Dashboard processing events/renderers need no new target outcome: this failure precedes serving and is a daemon/management diagnostic, not a completed Issue/PR execution. + +## Management recovery ownership evidence and one-shot issuance + +The prospective original-genesis producer, full-scope recovery-containment +observer, and `management/recovery_epoch.py` host issuer have separate authority +lifetimes. Genesis and containment diagnostics remain evidence-only. The current +observer emits `observation_only=True`, `recovery_grant=False`, and +`admission_created=False`; it never writes an operation or readiness outcome. +The issuer's actual append-only writer emits `recovery_epoch_phase=reserved`, +`intent`, `consumed`, and `receipt`, with request/target bindings and +`grant_delivered=False`. These events occur after their durable record/head +boundaries, and their Loguru source locations identify the real writer. A later +`grant-send-completed` observation means that one socket send completed; it does +not acknowledge delivery, launch a child, or establish current readiness. + +The explicit bootstrap custody origin logs `consumer-custody` only after a +verified complete offer and exact consumed grant. `consumer-unavailable` and host +`unavailable` retain `current_ready=False`. A shutdown signal while awaiting an +offer immediately fails that prelaunch endpoint. An exited issuer or lost fence +makes current custody unavailable without changing historical receipts or +inventing a recovery success. Both CLI origins use the existing Loguru stdout +and file sinks; the host file is `/var/log/auto-coder/recovery-epoch.log` and the +bootstrap file is in its separate target state's `logs` directory. + +These host commands do not enter an Issue/PR processing attempt, implementation +slot, management operation admission, dispatch, or worker lifecycle. There is +therefore no dashboard event-schema or renderer addition. Creating a dashboard +success/job for a non-authorizing observation or an unacknowledged grant would +misrepresent the boundary. The separate #2825 consumer must own any future child +readiness/admission projection. Existing unknown originals and their key/target +exclusions remain unknown and excluded. + +Run `bash scripts/test.sh --ci-tests-only + tests/test_management_recovery_epoch_observability.py + tests/test_management_recovery_epoch_state.py + tests/test_management_recovery_epoch_authorization.py + tests/test_management_recovery_epoch_join.py + tests/test_management_recovery_epoch_peer.py + tests/test_management_recovery_epoch_consumer.py + tests/test_management_recovery_containment_resources.py`. +The observability test drives the real durable writer in a separate process and +checks the ordered structured phases, source locations, stdout and file output, +and absent grant/readiness claims. The real-signal consumer cases observe the +pre-offer barrier, cancellation, no custody-acknowledgment success, no target +store/child creation and released target exclusion. Dataclass join fixtures are +component scope checks, not evidence of native containment. The independent +systemd/Docker VM workflow `Management Recovery Epoch Acceptance` must traverse +actual original producers, controller/kernel observations, connected peer +custody and at-most-one issuance for integrated acceptance. See +`client-features/management-recovery-epoch.md`, +`client-features/management-original-genesis.md`, and +`client-features/management-recovery-containment-observation.md` for the fixed +profiles, interfaces and source-of-truth limits. diff --git a/scripts/prepare_management_test_image.py b/scripts/prepare_management_test_image.py index c7659fc28..fdc015187 100644 --- a/scripts/prepare_management_test_image.py +++ b/scripts/prepare_management_test_image.py @@ -80,7 +80,7 @@ def prepare_image(repository: Path, github_env: Path | None = None) -> str: encoded = base64.b64encode(serialize_build_identity(identity).encode()).decode() tag = f"auto-coder:management-fixture-{identity.source_revision[:12]}" dockerfile = (repository / "Dockerfile").read_text() + ( - "\nFROM python-environment AS management-test-fixture\n" 'ENV PATH="/opt/auto-coder/.venv/bin:${PATH}"\n' "ARG AUTO_CODER_SOURCE_REVISION\n" "RUN AUTO_CODER_SOURCE_REVISION=$AUTO_CODER_SOURCE_REVISION /opt/auto-coder/.venv/bin/python -m auto_coder.build_provenance embed\n" + "\nFROM python-environment AS management-test-fixture\n" 'ENV PATH="/opt/auto-coder/.venv/bin:${PATH}"\n' "ARG AUTO_CODER_SOURCE_REVISION\n" "RUN AUTO_CODER_SOURCE_REVISION=$AUTO_CODER_SOURCE_REVISION /opt/auto-coder/.venv/bin/python -B -m auto_coder.build_provenance embed\n" ) subprocess.run( ["docker", "build", "--file", "-", "--target", "management-test-fixture", "--build-arg", f"AUTO_CODER_BUILD_IDENTITY_B64={encoded}", "--build-arg", f"AUTO_CODER_SOURCE_REVISION={identity.source_revision}", "--tag", tag, str(repository)], diff --git a/scripts/recovery_epoch_containment_fixture.py b/scripts/recovery_epoch_containment_fixture.py new file mode 100644 index 000000000..b11eeb544 --- /dev/null +++ b/scripts/recovery_epoch_containment_fixture.py @@ -0,0 +1,189 @@ +"""Real-object helpers for the separately authorized disposable recovery VM. + +The calling harness first runs require_disposable_host on the initially empty +VM. These helpers retain live original namespaces, capture actual records and +install an explicitly reviewed fixture scope; the production observer never +imports this module and never performs these fixture effects. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Protocol + +from scripts.test_management_containment_closed_host import MARKER, WatcherFixture, read_authorization +from scripts.test_management_containment_enrollment_host import _private_directory, _write_private + +from auto_coder.management import proxy_watch_cli +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.containment_kernel_subobservation import _holder_start_token, _require_capture_host +from auto_coder.management.containment_observation import observe_recovery_docker +from auto_coder.management.containment_route_manifest import observe_recovery_route_manifest, observe_selected_bind_roots, propose_unapproved_route_draft +from auto_coder.management.fresh_host import _write_evidence +from auto_coder.management.fresh_host_observation import _engine_id +from auto_coder.management.installed_bootstrap_mount_profile import _inspect +from auto_coder.management.original_attempts import canonical, source_identity +from auto_coder.management.recovery_containment import capture_recovery_original_pair, load_enrollment, route_selection +from auto_coder.management.recovery_containment_model import RECOVERY_ENROLLMENT_ROOT, RECOVERY_PROFILE, RecoveryEnrollment, RecoveryEnrollmentObservation, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryTarget, RecoveryWatcher +from auto_coder.management.recovery_containment_profile import _mapped_file, read_bound_file + + +class Generation(Protocol): + name: str + root: Path + container: str + proxy: str | None + + def path(self, name: str) -> Path: ... + + +class OriginalChain(Protocol): + image: str + proxy_image: str + + +def selected_file(path: Path, *, source: bool = False) -> RecoveryFile: + resolved = path.resolve(strict=True) + info = resolved.lstat() + result = RecoveryFile(str(resolved), hashlib.sha256(resolved.read_bytes()).hexdigest(), info.st_dev, info.st_ino) + read_bound_file(result, source=source) + return result + + +def selected_lock(path: Path) -> RecoveryLock: + if path != path.resolve(strict=True): + raise ValueError("fixture lock is not canonical") + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1: + raise ValueError("fixture lock is not the actual private original object") + return RecoveryLock(str(path), info.st_dev, info.st_ino) + + +@dataclass +class CapturedGeneration: + original: RecoveryOriginal + namespace_fd: int + + def close(self) -> None: + if self.namespace_fd >= 0: + os.close(self.namespace_fd) + self.namespace_fd = -1 + + +def capture_generation(chain: OriginalChain, generation: Generation, *, generation_id: str | None = None) -> CapturedGeneration: + """Capture while the genuine manager and paired Caddy are still alive.""" + _require_capture_host() + if generation.proxy is None: + raise ValueError("the original paired Caddy must already exist") + mounts = observe_selected_bind_roots((generation.container, generation.proxy)) + bootstrap = selected_file(generation.path("bootstrap") / "config.json") + config = json.loads(read_bound_file(bootstrap)) + management_destination = config.get("management_config") + if management_destination is None: + runtime = next(item for item in config["runtimes"] if item["reference"] == config["initial_runtime"]) + arguments = runtime["arguments"] + management_destination = arguments[arguments.index("--config") + 1] + manager_mounts = tuple(item for item in mounts if item.container_id == generation.container) + management = selected_file(Path(_mapped_file(management_destination, manager_mounts, writable=False))) + owner = selected_lock(generation.path("state") / ".supervisor-owner.lock") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + preliminary = RecoveryOriginal(generation_id or generation.name, config["instance_id"], generation.container, chain.image, generation.proxy, chain.proxy_image, "started", boot, bootstrap, management, selected_file(generation.path("Caddyfile")), mounts, owner, None, None) + actual = _inspect(generation.container) + state = actual.get("State") + if not isinstance(state, dict) or state.get("Running") is not True or type(state.get("Pid")) is not int or state["Pid"] <= 1: + raise ValueError("fixture original is not currently running") + descriptor = os.open(f"/proc/{state['Pid']}/ns/net", os.O_RDONLY | os.O_CLOEXEC) + try: + namespace = os.readlink(f"/proc/self/fd/{descriptor}") + info = os.fstat(descriptor) + captured_boot, manager, proxy = capture_recovery_original_pair(preliminary, source=source_identity()) + if captured_boot != boot or manager.net_namespace != namespace: + raise ValueError("fixture original namespace changed during capture") + directory = generation.path("recovery-original-capture") + directory.mkdir(mode=0o700) + _write_evidence(directory, {"management": asdict(manager), "proxy": asdict(proxy)}) + pin = OriginalNetworkPin(os.getpid(), _holder_start_token(os.getpid()), boot, descriptor, namespace, info.st_dev, info.st_ino) + complete = replace(preliminary, prior_evidence=selected_file(directory / "prior-evidence.json"), namespace_pin=pin) + parsed = RecoveryOriginal.parse(json.loads(json.dumps(asdict(complete)))) + return CapturedGeneration(parsed, descriptor) + except BaseException: + os.close(descriptor) + raise + + +def start_watcher(generation: Generation, original: RecoveryOriginal, *, first: bool = False) -> tuple[WatcherFixture, RecoveryWatcher]: + """Run, observe and inhibit the real watcher before retaining its EX lock.""" + unit = "auto-coder-management-proxy-watch" + ("" if first else "-" + generation.name) + ".service" + fixture = WatcherFixture.start(generation.root, original.management_id, original.management_image_id, original.proxy_id, original.proxy_image_id, Path(original.proxy_config.path), unit=unit) + selected = RecoveryWatcher(unit, selected_file(fixture.config), selected_file(Path(proxy_watch_cli.__file__), source=True), selected_lock(fixture.state / "proxy-watch.lock")) + return fixture, selected + + +def target_from_container(container_id: str, *, bootstrap_config: Path, management_config: Path, published_port: int) -> RecoveryTarget: + """Select the real already running prelaunch target and its created owner.""" + _require_capture_host() + inspected = _inspect(container_id) + config = json.loads(bootstrap_config.read_bytes()) + mounts = observe_selected_bind_roots((container_id,)) + state = next(item.source for item in mounts if item.destination == config["state_root"]) + image = inspected.get("Image") + if not isinstance(image, str): + raise ValueError("fixture target image is absent") + return RecoveryTarget.parse(json.loads(json.dumps(asdict(RecoveryTarget(config["instance_id"], container_id, image, selected_file(bootstrap_config), selected_file(management_config), mounts, selected_lock(Path(state) / ".supervisor-owner.lock"), published_port))))) + + +def enroll_recovery_scope( + captures: tuple[CapturedGeneration, ...], + target: RecoveryTarget, + watchers: tuple[RecoveryWatcher, ...], + *, + deployment_id: str, + enrollment_id: str, + authorization: dict[str, object], +) -> RecoveryEnrollmentObservation: + """Install fixture approval only under the still-current separate VM marker. + + The caller owns complete controller setup/inhibition. The route draft is + inspected here and marked approved only after all detected candidate and + explicitly selected watcher routes already have their real inhibited facts. + This helper neither stops controllers nor treats an unapproved list as proof. + """ + _require_capture_host() + actual_authorization = read_authorization(MARKER) + if actual_authorization != authorization or authorization["boot_id"] != Path("/proc/sys/kernel/random/boot_id").read_text().strip() or authorization["machine_id"] != Path("/etc/machine-id").read_text().strip() or authorization["engine_id"] != _engine_id(): + raise PermissionError("fixture authorization no longer selects this exact VM") + selected = RecoveryEnrollment( + "approved-management-recovery-containment-v1", + deployment_id, + enrollment_id, + str(authorization["authorization_id"]), + RECOVERY_PROFILE, + str(authorization["engine_id"]), + source_identity(), + tuple(sorted((item.original for item in captures), key=lambda item: item.generation_id)), + target, + tuple(sorted(watchers, key=lambda item: item.unit)), + True, + ) + RecoveryEnrollment.parse(json.loads(json.dumps(asdict(selected)))) + _private_directory(RECOVERY_ENROLLMENT_ROOT) + path = RECOVERY_ENROLLMENT_ROOT / (enrollment_id + ".json") + route_path = RECOVERY_ENROLLMENT_ROOT / (enrollment_id + ".routes.json") + if path.exists() or path.is_symlink() or route_path.exists() or route_path.is_symlink(): + raise FileExistsError("fixture recovery approval already exists") + _write_private(path, canonical(asdict(selected))) + approved = load_enrollment(enrollment_id) + managers = tuple(item.management_id for item in selected.originals) + (selected.target.management_id,) + docker = observe_recovery_docker(managers, tuple(item.proxy_id for item in selected.originals)) + draft = propose_unapproved_route_draft(route_selection(approved), docker, closed_profile=True) + # The isolated host harness has already audited its stock routes. This is + # the explicit separately authorized fixture approval, not observer output. + draft["complete_ordinary_routes_attested"] = True + _write_private(route_path, canonical(draft)) + observe_recovery_route_manifest(route_selection(approved), docker) + return approved diff --git a/scripts/recovery_epoch_fault_tracer.py b/scripts/recovery_epoch_fault_tracer.py new file mode 100644 index 000000000..a63510597 --- /dev/null +++ b/scripts/recovery_epoch_fault_tracer.py @@ -0,0 +1,335 @@ +"""Real Linux syscall barriers for the disposable recovery acceptance fixture. + +This observer never changes syscall arguments/results or supplies production +observations. It stops the actual issuer at an exclusive publication open or +at the successful directory fsync following its real file fsync. The caller +then kills the pinned process or resumes it after an external fixture mutation. +Linux ptrace ABI: https://man7.org/linux/man-pages/man2/ptrace.2.html +""" + +from __future__ import annotations + +import ctypes +import hashlib +import os +import platform +import signal +import subprocess +import threading +from dataclasses import dataclass +from pathlib import Path + +from auto_coder.logger_config import get_logger + +logger = get_logger(__name__) +PHASES = ("reserved", "intent", "consumed", "receipt") +PUBLICATIONS = tuple(phase + suffix for phase in PHASES for suffix in (".json", "-head.json")) +BOUNDARY_SCENARIOS = tuple(f"{when}-{name.removesuffix('.json')}" for name in PUBLICATIONS for when in ("before", "after")) + +_PTRACE_SEIZE = 0x4206 +_PTRACE_INTERRUPT = 0x4207 +_PTRACE_GET_SYSCALL_INFO = 0x420E +_PTRACE_SYSCALL = 24 +_PTRACE_DETACH = 17 +_PTRACE_O_TRACESYSGOOD = 1 +_PTRACE_O_EXITKILL = 1 << 20 +_WAIT_ALL = 0x40000000 +_AUDIT_ARCH_X86_64 = 0xC000003E +_SYS_OPENAT = 257 +_SYS_FSYNC = 74 + + +class _Entry(ctypes.Structure): + _fields_ = [("number", ctypes.c_uint64), ("arguments", ctypes.c_uint64 * 6)] + + +class _Exit(ctypes.Structure): + _fields_ = [("result", ctypes.c_int64), ("is_error", ctypes.c_uint8)] + + +class _Call(ctypes.Union): + _fields_ = [("entry", _Entry), ("exit", _Exit), ("space", ctypes.c_uint8 * 64)] + + +class _Information(ctypes.Structure): + _fields_ = [("operation", ctypes.c_uint8), ("padding", ctypes.c_uint8 * 3), ("architecture", ctypes.c_uint32), ("instruction", ctypes.c_uint64), ("stack", ctypes.c_uint64), ("call", _Call)] + + +def parse_boundary(scenario: str) -> tuple[str, str]: + if scenario not in BOUNDARY_SCENARIOS: + raise ValueError("a reviewed publication boundary scenario is required") + when, name = scenario.split("-", 1) + return name + ".json", when + + +def _token(pid: int) -> str: + return Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[19] + + +@dataclass(frozen=True) +class PublicationObservation: + process_id: int + start_token: str + boot_id: str + name: str + when: str + directory_device: int + directory_inode: int + file_device: int | None + file_inode: int | None + file_sha256: str | None + file_fsync_observed: bool + directory_fsync_observed: bool + + +class KernelPublicationBarrier: + """Attach to one actual child and retain its exact syscall stop. + + Do not call Popen.poll/wait between start and completion: this dedicated + tracer thread exclusively consumes waitpid notifications during that time. + start returns with the issuer stopped. resume_initial releases that initial + stop. wait returns with the selected native publication still stopped. + kill or release finishes tracing; close kills an unfinished tracee. + """ + + def __init__(self, issuer: subprocess.Popen, *, root: Path, name: str, when: str): + if platform.system() != "Linux" or platform.machine() != "x86_64": + raise ValueError("the native publication fixture requires Linux x86_64") + if name not in PUBLICATIONS or when not in {"before", "after"}: + raise ValueError("unsupported publication boundary") + if not root.is_absolute() or root.resolve(strict=True) != root or not root.is_dir(): + raise ValueError("the publication directory must be original and canonical") + self.issuer = issuer + self.root = root + self.name = name + self.when = when + self.root_identity = (root.stat().st_dev, root.stat().st_ino) + self.start_token = _token(issuer.pid) + self.boot_id = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + self._pidfd = os.pidfd_open(issuer.pid) + self._ready = threading.Event() + self._resume = threading.Event() + self._hit = threading.Event() + self._action = threading.Event() + self._done = threading.Event() + self._decision = "kill" + self._error: BaseException | None = None + self._observation: PublicationObservation | None = None + self._thread: threading.Thread | None = None + self._file_identity: tuple[int, int] | None = None + self._file_synced = False + self._pending: str | None = None + self._libc = ctypes.CDLL(None, use_errno=True) + self._libc.ptrace.argtypes = [ctypes.c_ulong, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] + self._libc.ptrace.restype = ctypes.c_long + + def _ptrace(self, request: int, address: int = 0, data: int | ctypes.c_void_p = 0) -> int: + result = self._libc.ptrace(request, self.issuer.pid, ctypes.c_void_p(address), ctypes.c_void_p(data) if isinstance(data, int) else data) + if result == -1: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + return result + + def _require_lifetime(self) -> None: + if _token(self.issuer.pid) != self.start_token or Path("/proc/sys/kernel/random/boot_id").read_text().strip() != self.boot_id: + raise AssertionError("the traced issuer lifetime changed") + if (self.root.stat().st_dev, self.root.stat().st_ino) != self.root_identity: + raise AssertionError("the traced publication directory changed") + + def _descriptor_identity(self, descriptor: int) -> tuple[int, int]: + info = Path(f"/proc/{self.issuer.pid}/fd/{descriptor}").stat() + return info.st_dev, info.st_ino + + def _cstring(self, address: int) -> str: + with Path(f"/proc/{self.issuer.pid}/mem").open("rb", buffering=0) as memory: + value = bytearray() + for offset in range(4096): + byte = os.pread(memory.fileno(), 1, address + offset) + if byte == b"\0": + return value.decode("utf-8", "strict") + if len(byte) != 1: + break + value.extend(byte) + raise AssertionError("the native publication pathname was not bounded") + + def _observe_call(self) -> bool: + info = _Information() + length = self._ptrace(_PTRACE_GET_SYSCALL_INFO, ctypes.sizeof(info), ctypes.cast(ctypes.byref(info), ctypes.c_void_p)) + if length < 24 or info.architecture != _AUDIT_ARCH_X86_64: + raise AssertionError("the native syscall information is unavailable") + if info.operation == 1: + self._pending = None + call = info.call.entry + if call.number == _SYS_OPENAT and call.arguments[2] & (os.O_CREAT | os.O_EXCL) == os.O_CREAT | os.O_EXCL: + directory = ctypes.c_int(call.arguments[0]).value + if directory >= 0 and self._descriptor_identity(directory) == self.root_identity and self._cstring(call.arguments[1]) == self.name: + if self._file_identity is not None: + raise AssertionError("the same native publication was attempted twice") + if (self.root / self.name).exists(): + raise AssertionError("the selected publication already existed before exclusive creation") + self._pending = "open" + return self.when == "before" + elif call.number == _SYS_FSYNC and self._file_identity is not None: + identity = self._descriptor_identity(call.arguments[0]) + if identity == self._file_identity: + self._pending = "file-fsync" + elif identity == self.root_identity and self._file_synced: + self._pending = "directory-fsync" + elif info.operation == 2 and self._pending is not None: + pending, self._pending = self._pending, None + if info.call.exit.is_error or info.call.exit.result < 0: + raise AssertionError("the selected real publication syscall failed") + if pending == "open": + self._file_identity = self._descriptor_identity(info.call.exit.result) + actual = (self.root / self.name).stat() + if self._file_identity != (actual.st_dev, actual.st_ino): + raise AssertionError("the created publication differs from its actual descriptor") + elif pending == "file-fsync": + self._file_synced = True + elif pending == "directory-fsync": + return self.when == "after" + return False + + def _record(self) -> None: + self._require_lifetime() + path = self.root / self.name + device = inode = digest = None + if self.when == "after": + actual = path.stat() + device, inode = actual.st_dev, actual.st_ino + if self._file_identity != (device, inode) or not self._file_synced: + raise AssertionError("actual publication/file fsync identity is missing") + digest = hashlib.sha256(path.read_bytes()).hexdigest() + elif path.exists(): + raise AssertionError("the before-publication barrier was crossed") + self._observation = PublicationObservation(self.issuer.pid, self.start_token, self.boot_id, self.name, self.when, *self.root_identity, device, inode, digest, self._file_synced, self.when == "after") + logger.bind(native_publication=self.name, native_boundary=self.when, issuer_pid=self.issuer.pid).info("Actual issuer stopped at native publication boundary") + self._hit.set() + + def _wait_status(self) -> int: + pid, status = os.waitpid(self.issuer.pid, _WAIT_ALL) + if pid != self.issuer.pid: + raise AssertionError("a different process supplied the native trace status") + if os.WIFEXITED(status) or os.WIFSIGNALED(status): + self.issuer.returncode = os.waitstatus_to_exitcode(status) + return status + + def _run(self) -> None: + attached = False + try: + self._require_lifetime() + self._ptrace(_PTRACE_SEIZE, data=_PTRACE_O_TRACESYSGOOD | _PTRACE_O_EXITKILL) + attached = True + self._ptrace(_PTRACE_INTERRUPT) + status = self._wait_status() + if not os.WIFSTOPPED(status): + raise AssertionError("actual issuer exited before the initial trace barrier") + self._ready.set() + if not self._resume.wait(180): + raise TimeoutError("the actual target probe did not release the initial barrier") + self._ptrace(_PTRACE_SYSCALL) + while True: + status = self._wait_status() + if not os.WIFSTOPPED(status): + raise AssertionError("actual issuer exited before the selected publication boundary") + delivered = os.WSTOPSIG(status) + if delivered == signal.SIGTRAP | 0x80: + if self._observe_call(): + self._record() + if not self._action.wait(180): + raise TimeoutError("the observed publication was not explicitly released or killed") + if self._decision == "release": + self._ptrace(_PTRACE_DETACH) + attached = False + return + signal.pidfd_send_signal(self._pidfd, signal.SIGKILL) + while self.issuer.returncode is None: + self._wait_status() + return + delivered = 0 + elif delivered == signal.SIGTRAP and status >> 16: + delivered = 0 + self._ptrace(_PTRACE_SYSCALL, data=delivered) + except BaseException as error: + self._error = error + finally: + if attached and self.issuer.returncode is None: + try: + signal.pidfd_send_signal(self._pidfd, signal.SIGKILL) + while self.issuer.returncode is None: + self._wait_status() + except (OSError, ChildProcessError): + pass + self._ready.set() + self._hit.set() + self._done.set() + + def _raise_error(self) -> None: + if self._error is not None: + raise self._error + + def start(self) -> None: + if self._thread is not None: + raise ValueError("a native publication tracer cannot be restarted") + self._thread = threading.Thread(target=self._run, name="recovery-native-publication", daemon=True) + self._thread.start() + if not self._ready.wait(15): + self.close() + raise TimeoutError("the native initial trace barrier was not observed") + self._raise_error() + + def resume_initial(self) -> None: + if not self._ready.is_set() or self._done.is_set(): + self._raise_error() + raise ValueError("no active initial publication barrier") + self._resume.set() + + def wait(self, timeout: float = 900) -> PublicationObservation: + if not self._hit.wait(timeout): + self.close() + raise TimeoutError("the actual issuer never reached the selected publication") + self._raise_error() + if self._observation is None: + raise AssertionError("the native publication observation is missing") + return self._observation + + def _finish(self, action: str) -> None: + self._raise_error() + if self._observation is None or self._done.is_set(): + raise ValueError("the selected native publication is not currently stopped") + self._decision = action + self._action.set() + if not self._done.wait(15): + self.close() + raise TimeoutError("the native publication trace did not finish") + self._raise_error() + + def kill(self) -> None: + self._finish("kill") + + def release(self) -> None: + self._finish("release") + + def close(self) -> None: + if self._pidfd < 0: + return + if self._thread is not None and not self._done.is_set(): + self._decision = "kill" + self._resume.set() + self._action.set() + try: + signal.pidfd_send_signal(self._pidfd, signal.SIGKILL) + except ProcessLookupError: + pass + if not self._done.wait(15): + raise TimeoutError("the actual traced issuer did not exit during cleanup") + if self._thread is not None and self._done.is_set() and self._error is not None and self.issuer.returncode is None: + # A refused initial SEIZE never transfers waitpid ownership to the + # tracer. End the caller's exact attempted issuer here as well. + try: + signal.pidfd_send_signal(self._pidfd, signal.SIGKILL) + except ProcessLookupError: + pass + self.issuer.wait(timeout=15) + os.close(self._pidfd) + self._pidfd = -1 diff --git a/scripts/recovery_vm/Dockerfile b/scripts/recovery_vm/Dockerfile new file mode 100644 index 000000000..140742c15 --- /dev/null +++ b/scripts/recovery_vm/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:24.04 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates cloud-image-utils curl python3 qemu-system-x86 qemu-utils \ + && rm -rf /var/lib/apt/lists/* +ARG CLOUD_IMAGE_URL=https://cloud-images.ubuntu.com/minimal/releases/noble/release-20261105/ubuntu-24.04-minimal-cloudimg-amd64.img +ARG CLOUD_IMAGE_SHA256=406630c08b5e545fcc7209854b129d47e3af822db128c9f19336593b5a2d3fe0 +RUN curl --fail --location --retry 3 "$CLOUD_IMAGE_URL" -o /opt/base.img \ + && echo "$CLOUD_IMAGE_SHA256 /opt/base.img" | sha256sum --check --strict +COPY runner.sh /opt/runner.sh +ENTRYPOINT ["/bin/bash", "/opt/runner.sh"] diff --git a/scripts/recovery_vm/compose.yaml b/scripts/recovery_vm/compose.yaml new file mode 100644 index 000000000..4615fa828 --- /dev/null +++ b/scripts/recovery_vm/compose.yaml @@ -0,0 +1,34 @@ +services: + fixture: + build: + context: ${RECOVERY_VM_TOOLING:?Set the reviewed tooling directory} + environment: + APPROVED_AUTHORIZATION_ID: ${APPROVED_AUTHORIZATION_ID:?Set the matching disposable fixture authorization} + user: "${RECOVERY_VM_UID:?Set the preparation UID}:${RECOVERY_VM_GID:?Set the preparation GID}" + group_add: + - "${RECOVERY_KVM_GID:?Set the KVM device group}" + init: true + cpus: 2.0 + mem_limit: 6g + pids_limit: 256 + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + devices: + - /dev/kvm:/dev/kvm:rwm + tmpfs: + - /tmp:size=128m + volumes: + - type: bind + source: ${RECOVERY_VM_ROOT:?Set the prepared fixture directory}/input + target: /input + read_only: true + - type: bind + source: ${RECOVERY_VM_ROOT:?Set the prepared fixture directory}/disk + target: /disk + - type: bind + source: ${RECOVERY_VM_ROOT:?Set the prepared fixture directory}/output + target: /output + stop_grace_period: 30s diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh new file mode 100644 index 000000000..8d08c505e --- /dev/null +++ b/scripts/recovery_vm/guest.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +input=/mnt/fixture-input +output=/mnt/fixture-output +export PYTHONDONTWRITEBYTECODE=1 +exec > >(tee -a "$output/guest.log") 2>&1 + +finish() { + result=$? + trap - EXIT + set +e + rm -f /run/recovery-fixture-source-token + printf '%s\n' "$result" | tee "$output/guest-exit-code" > /dev/null + sync + systemctl poweroff --no-block + exit "$result" +} +trap finish EXIT +python3 - "$input" <<'PY' +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +manifest = json.loads((root / "manifest.json").read_text()) +if manifest.get("schema") != "auto-coder-recovery-vm-input-v1": + raise ValueError("Unsupported recovery fixture input") +for name, digest in manifest["inputs_sha256"].items(): + with (root / name).open("rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != digest: + raise ValueError("Recovery fixture input changed: " + name) +PY +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends ca-certificates cron curl docker.io git python3-venv +systemctl start docker +python3 -m venv /opt/fixture-tools +/opt/fixture-tools/bin/pip install uv==0.13.6 +mkdir -m 700 /opt/recovery-fixture /opt/reviewed-source +tar --extract --gzip --file "$input/source.tar.gz" --directory /opt/recovery-fixture +cd /opt/recovery-fixture +/opt/fixture-tools/bin/uv sync --locked --no-editable --no-dev --extra test +install -m 600 "$input/source-token" /run/recovery-fixture-source-token +mapfile -t selected < <(python3 - "$input" <<'PY' +import json +import pathlib +import sys + +manifest = json.loads((pathlib.Path(sys.argv[1]) / "manifest.json").read_text()) +for key in ("authorization_id", "manager_image", "proxy_image", "ready_source_sha", "scenario"): + print(manifest[key]) +PY +) +git -C /opt/reviewed-source init +git -C /opt/reviewed-source remote add origin https://github.com/kitamura-tetsuo/auto-coder.git +git -C /opt/reviewed-source fetch --no-tags origin "${selected[3]}" +git -C /opt/reviewed-source checkout --detach FETCH_HEAD +test "$(git -C /opt/reviewed-source rev-parse --is-shallow-repository)" = false +docker image load --input "$input/images.tar" +python3 - "$input" <<'PY' +import json +import os +import pathlib +import subprocess +import sys + +manifest = json.loads((pathlib.Path(sys.argv[1]) / "manifest.json").read_text()) +marker = { + "schema": "auto-coder-disposable-containment-host-v1", + "authorization_id": manifest["authorization_id"], + "machine_id": pathlib.Path("/etc/machine-id").read_text().strip(), + "boot_id": pathlib.Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "engine_id": subprocess.check_output(["docker", "info", "--format", "{{.ID}}"], text=True).strip(), + "synthetic_only": True, +} +fd = os.open("/etc/auto-coder-disposable-containment-host.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) +with os.fdopen(fd, "w") as stream: + json.dump(marker, stream, sort_keys=True) +PY +export AUTOCODER_FRESH_TEST_BASE_IMAGE="${selected[1]}" +export AUTOCODER_RECOVERY_CADDY_IMAGE="${selected[2]}" +export PATH="/opt/fixture-tools/bin:$PATH" +.venv/bin/python -m scripts.test_management_recovery_epoch_host \ + --authorization-id "${selected[0]}" \ + --source-repository /opt/reviewed-source \ + --source-token-file /run/recovery-fixture-source-token \ + --source-sha "${selected[3]}" \ + --scenario "${selected[4]}" \ + --uv /opt/fixture-tools/bin/uv --uv-version 0.13.6 \ + --output "$output/recovery-epoch-result.json" diff --git a/scripts/recovery_vm/prepare.sh b/scripts/recovery_vm/prepare.sh new file mode 100644 index 000000000..b8058c040 --- /dev/null +++ b/scripts/recovery_vm/prepare.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +# Export exact reviewed source/images; preparation does not launch a guest. +if [[ "$#" != 6 || ! "$1" =~ ^/var/tmp/ac-recovery-vm-[a-zA-Z0-9_-]+$ || ! "$2" =~ ^[a-zA-Z0-9_.:-]{1,128}$ || ! "$3" =~ ^sha256:[0-9a-f]{64}$ || ! "$4" =~ ^sha256:[0-9a-f]{64}$ || ! "$6" =~ ^(grant|kill-after-intent|kill-after-consume|lost-reply)$ ]]; then + echo 'Usage: prepare.sh /var/tmp/ac-recovery-vm-UNIQUE AUTHORIZATION_ID MANAGER_IMAGE_ID CADDY_IMAGE_ID READ_TOKEN_FILE SCENARIO' >&2 + exit 2 +fi +fixture_root=$1 +fixture_authorization=$2 +manager_image=$3 +proxy_image=$4 +source_token_file=$5 +fixture_scenario=$6 +tooling_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) +repo_root=$(git -C "$tooling_root" rev-parse --show-toplevel) +test -c /dev/kvm +test -f "$source_token_file" +if [[ -e "$fixture_root" || -L "$fixture_root" ]]; then + echo 'The recovery fixture directory must not exist.' >&2 + exit 2 +fi +if [[ -n "$(git -C "$repo_root" status --porcelain --untracked-files=normal)" ]]; then + echo 'Commit reviewed source before exporting the recovery VM inputs.' >&2 + exit 2 +fi +for selected in "$manager_image" "$proxy_image"; do + test "$(docker image inspect --format '{{.Id}}' "$selected")" = "$selected" +done +mkdir -m 700 "$fixture_root" "$fixture_root/input" "$fixture_root/output" "$fixture_root/disk" +exec > >(tee -a "$fixture_root/output/preparation.log") 2>&1 +git -C "$repo_root" archive --format=tar.gz HEAD | tee "$fixture_root/input/source.tar.gz" > /dev/null +docker image save "$manager_image" "$proxy_image" | tee "$fixture_root/input/images.tar" > /dev/null +cp "$tooling_root/guest.sh" "$fixture_root/input/guest.sh" +install -m 600 "$source_token_file" "$fixture_root/input/source-token" +fixture_commit=$(git -C "$repo_root" rev-parse HEAD) +python3 - "$fixture_root" "$fixture_authorization" "$fixture_commit" "$manager_image" "$proxy_image" "$fixture_scenario" <<'PY' +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +inputs = root / "input" +manifest = dict(zip(("authorization_id", "commit", "manager_image", "proxy_image", "scenario"), sys.argv[2:])) +manifest["schema"] = "auto-coder-recovery-vm-input-v1" +manifest["ready_source_sha"] = "e742338b3f311e6d30a22a8f5341b8072513d813" +manifest["inputs_sha256"] = {} +for name in ("source.tar.gz", "images.tar", "guest.sh"): + with (inputs / name).open("rb") as stream: + manifest["inputs_sha256"][name] = hashlib.file_digest(stream, "sha256").hexdigest() +(inputs / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") +(inputs / "meta-data").write_text("instance-id: " + manifest["authorization_id"] + "\nlocal-hostname: ac-recovery-fixture\n") +PY +tee "$fixture_root/input/user-data" >/dev/null <<'YAML' +#cloud-config +users: [] +disable_root: true +ssh_pwauth: false +runcmd: + - [mkdir, -p, /mnt/fixture-input, /mnt/fixture-output] + - [mount, -t, 9p, -o, "trans=virtio,version=9p2000.L,ro", fixture-input, /mnt/fixture-input] + - [mount, -t, 9p, -o, "trans=virtio,version=9p2000.L", fixture-output, /mnt/fixture-output] + - [bash, /mnt/fixture-input/guest.sh] +power_state: + mode: poweroff + delay: now + timeout: 30 + condition: true +YAML +tee "$fixture_root/compose.env" >/dev/null <&2 + exit 2 +fi +if [[ -e /disk/guest.qcow2 ]]; then + echo 'A used guest disk cannot be reused as fresh recovery evidence.' >&2 + exit 2 +fi +cloud-localds /disk/seed.img /input/user-data /input/meta-data +qemu-img create -f qcow2 -F qcow2 -b /opt/base.img /disk/guest.qcow2 32G +set +e +timeout --signal=TERM --kill-after=30s 120m qemu-system-x86_64 \ + -accel kvm -cpu host -smp 2 -m 4608 -nodefaults -no-reboot \ + -display none -serial stdio -monitor none \ + -drive file=/disk/guest.qcow2,if=virtio,format=qcow2 \ + -drive file=/disk/seed.img,if=virtio,format=raw,readonly=on \ + -netdev user,id=public-bootstrap -device virtio-net-pci,netdev=public-bootstrap \ + -virtfs local,path=/input,mount_tag=fixture-input,security_model=none,readonly=on \ + -virtfs local,path=/output,mount_tag=fixture-output,security_model=none \ + 2>&1 | tee /output/serial.log +qemu_status=${PIPESTATUS[0]} +set -e +if [[ "$qemu_status" != 0 || ! -f /output/guest-exit-code ]]; then + echo "The recovery guest did not complete; QEMU exit status: $qemu_status" >&2 + exit 1 +fi +result=$(cat /output/guest-exit-code) +if [[ "$result" != 0 ]]; then + echo "The recovery acceptance driver failed with status $result." >&2 + exit 1 +fi +python3 - <<'PY' +import json +from pathlib import Path + +result = json.loads(Path("/output/recovery-epoch-result.json").read_text()) +manifest = json.loads(Path("/input/manifest.json").read_text()) +if result.get("status") != "passed" or result.get("scenario") != manifest["scenario"]: + raise SystemExit("The native recovery result does not record complete acceptance") +PY diff --git a/scripts/test_management_containment_closed_host.py b/scripts/test_management_containment_closed_host.py index 028a83827..0c691710e 100644 --- a/scripts/test_management_containment_closed_host.py +++ b/scripts/test_management_containment_closed_host.py @@ -462,45 +462,51 @@ class WatcherFixture: config: Path state: Path stdout: Path + unit: str = WATCHER_UNIT @classmethod - def start(cls, root: Path, manager: str, manager_image: str, proxy: str, proxy_image: str, caddyfile: Path) -> WatcherFixture: + def start(cls, root: Path, manager: str, manager_image: str, proxy: str, proxy_image: str, caddyfile: Path, *, unit: str = WATCHER_UNIT) -> WatcherFixture: + if re.fullmatch(r"auto-coder-management-proxy-watch(?:-[A-Za-z0-9_-]+)?\.service", unit) is None: + raise ValueError("fixture watcher unit must use a bounded management name") state = root / "watcher-state" state.mkdir(mode=0o700) config, output = root / "watcher-config.json", root / "watcher-output.log" _write_private(config, _json({"state_root": str(state), "manager_id": manager, "manager_image": manager_image, "proxy_id": proxy, "proxy_image": proxy_image, "config_sha256": hashlib.sha256(caddyfile.read_bytes()).hexdigest()})) _write_private(output, b"") - source = Path("/run/systemd/system") / WATCHER_UNIT + source = Path("/run/systemd/system") / unit command = f"{sys.executable} -I -B -m auto_coder.management.proxy_watch_cli --config {config}" if any(character in command for character in ("\n", "\r", "%", '"')) or " " in sys.executable or " " in str(root): raise ValueError("fixture watcher command contains unsupported systemd syntax") content = f"[Unit]\nDescription=Disposable original management proxy watcher\n[Service]\nType=simple\nExecStart={command}\nRestart=no\nStandardOutput=append:{output}\nStandardError=append:{output}\n".encode() _write_private(source, content) - result = cls(source, source.lstat().st_ino, hashlib.sha256(content).hexdigest(), config, state, output) + result = cls(source, source.lstat().st_ino, hashlib.sha256(content).hexdigest(), config, state, output, unit) try: _systemctl("daemon-reload") - _systemctl("start", WATCHER_UNIT) + _systemctl("start", unit) deadline = time.monotonic() + 30 while "ATTACHED\n" not in output.read_text(): if time.monotonic() >= deadline: raise AssertionError("the real installed watcher never observed Caddy attached") time.sleep(0.1) - pid = subprocess.run(("systemctl", "show", "--property=MainPID", "--value", WATCHER_UNIT), check=True, capture_output=True, text=True, timeout=10).stdout.strip() + pid = subprocess.run(("systemctl", "show", "--property=MainPID", "--value", unit), check=True, capture_output=True, text=True, timeout=10).stdout.strip() if not pid.isdigit() or int(pid) <= 1: raise AssertionError("the real watcher has no live unit-owned process") - _systemctl("stop", WATCHER_UNIT) - _systemctl("mask", WATCHER_UNIT) - observe_watcher_unit() + _systemctl("stop", unit) + _systemctl("mask", unit) + if unit == WATCHER_UNIT: + observe_watcher_unit() + elif not _systemd_unit(unit).inhibited: + raise AssertionError("the real additional watcher was not inhibited") return result except BaseException: result.cleanup() raise def cleanup(self) -> None: - _systemctl("stop", WATCHER_UNIT) - mask = Path("/etc/systemd/system") / WATCHER_UNIT + _systemctl("stop", self.unit) + mask = Path("/etc/systemd/system") / self.unit if mask.is_symlink() and os.readlink(mask) == "/dev/null": - _systemctl("unmask", WATCHER_UNIT) + _systemctl("unmask", self.unit) elif mask.exists() or mask.is_symlink(): raise AssertionError("fixture watcher mask was externally replaced") if self.source.lstat().st_ino != self.inode or hashlib.sha256(self.source.read_bytes()).hexdigest() != self.digest: diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py new file mode 100644 index 000000000..ad5cb98e2 --- /dev/null +++ b/scripts/test_management_recovery_epoch_host.py @@ -0,0 +1,749 @@ +"""Actual recovery ownership issuance on a separately authorized disposable VM. + +The driver uses original producers, the ordinary local Docker engine, systemd, +real namespace and flock custody, authenticated source publication and installed +READY verification. It never supplies an observation verdict or patches a +production function. The target stops at the explicit prelaunch custody mode; +consumer launch/recovery after this grant belongs to the separate consumer issue. +""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +import os +import re +import signal +import socket +import sqlite3 +import stat +import subprocess +import sys +import time +import uuid +from contextlib import ExitStack +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Callable + +from scripts.recovery_epoch_containment_fixture import CapturedGeneration, capture_generation, enroll_recovery_scope, selected_file, start_watcher, target_from_container +from scripts.test_management_containment_closed_host import InhibitedStockRoutes, WatcherFixture, require_disposable_host +from scripts.test_management_containment_enrollment_host import DisposableSystemdRoutes, _private_directory, _write_private + +from auto_coder.logger_config import get_logger, setup_logger +from auto_coder.management.bootstrap_supervisor import BootstrapConfig +from auto_coder.management.bootstrap_upgrade import _source_authority +from auto_coder.management.fresh_initialization import FreshInitializationIntent, claim_fresh_initialization, read_fresh_status +from auto_coder.management.operation_store import ManagementOperationStore +from auto_coder.management.operations import ActionParameter, ManagedTarget, OperationIntent, OperationPhase +from auto_coder.management.original_attempt_evidence import derive_lineage, observe, verify_lineage +from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, source_identity +from auto_coder.management.recovery_containment_model import RecoveryWatcher +from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, AUTHORIZATION_ROOT, AUTHORIZATION_SCHEMA, PEER_ROOT, PEER_SOCKET, EpochAuthorization +from auto_coder.management.recovery_epoch_consumer import ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource +from auto_coder.management.recovery_epoch_state import EPOCH_ROOT, EpochInventory +from auto_coder.management.self_update_handoff import runtime_for_receipt +from auto_coder.management.version_staging import ManagementPrepareIntent, ManagementReadyReceipt, ManagementRuntimeConfig, ManagementVersionStager +from auto_coder.operating_policy import MANAGEMENT_REPOSITORY +from tests.test_management_archived_cutover_container import Chain, Generation, diagnostic_log_lines + +logger = get_logger(__name__) +FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") +RESULT_SCHEMA = "auto-coder-native-recovery-epoch-result-v1" +SCENARIOS = ("grant", "kill-after-intent", "kill-after-consume", "lost-reply") +REQUIRED_CASES = frozenset( + { + "affirmative-original-genesis", + "native-successor-ready", + "native-failed-branch-retained", + "native-unresolved-successor-retained", + "missing-original-refused", + "genuine-installed-ready-in-target", + "independent-issuer-contender-refused", + "complete-native-epoch-granted", + "old-owner-and-controller-contenders-refused", + "issuer-death-ended-current-custody", + "consumed-epoch-replay-refused", + "originals-oauth-and-workers-preserved", + "no-consumer-child-or-admission", + } +) + + +def required_cases(scenario: str) -> frozenset[str]: + if scenario not in SCENARIOS: + raise ValueError("unsupported native recovery fixture scenario") + if scenario == "grant": + return REQUIRED_CASES + return (REQUIRED_CASES - {"complete-native-epoch-granted", "old-owner-and-controller-contenders-refused", "issuer-death-ended-current-custody", "consumed-epoch-replay-refused"}) | {"native-" + scenario, "interrupted-epoch-replay-refused"} + + +class WorkerFixture(Chain): + """Reuse the real worker producer in roots unrelated to management.""" + + def __init__(self, root: Path, image: str): + self.root = root + self.image = image + self.worker_compose = None + self.worker_environment = None + self.worker_baseline = None + self.start_workers() + code = "from auto_coder.management.worker_network_client import WorkerNetworkClient,WorkerEndpointConfig;c=WorkerNetworkClient(WorkerEndpointConfig.environment('beta-agent-table-worker'));assert c.inspect_supervisor().result_code=='OK';assert c.observe_worker().result_code=='OK'" + self.docker("exec", self.worker_manager, "python", "-B", "-c", code) + + def close(self) -> None: + if self.worker_compose is not None: + self.compose("down", "--volumes", "--remove-orphans") + + def preserved(self, expected: dict[str, object], posts: dict[str, int], claims: str) -> None: + assert self.workers() == expected, "an unrelated real worker lifetime changed" + assert self.effect_posts() == posts, "an unrelated real worker received a new effect POST" + assert _worker_claims(self.root / "workers") == claims, "an unrelated worker claim changed" + + def effect_posts(self) -> dict[str, int]: + return {channel: len(path.read_text().splitlines()) if path.exists() else 0 for channel in ("beta", "release", "agent-table") for path in (self.root / "workers" / channel / "fixture-posts",)} + + +def _digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _snapshot(roots: tuple[Path, ...]) -> str: + """Fingerprint protected original bytes/objects without exporting their data.""" + entries = [] + for root in roots: + for path in (root, *sorted(root.rglob("*"))): + info = path.lstat() + if path.is_symlink(): + content = os.readlink(path) + elif path.is_file(): + content = _digest(path) + elif path.is_dir(): + content = "directory" + elif stat.S_ISSOCK(info.st_mode): + content = "socket" + else: + raise AssertionError("unsupported original fixture object") + entries.append((str(path), info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_size, info.st_mtime_ns, info.st_ctime_ns, content)) + return hashlib.sha256(canonical(entries)).hexdigest() + + +def _worker_claims(root: Path) -> str: + # The actual supervisor refreshes its observation timestamp and atomically + # replaces state.json during a GET. Compare its stable operation/ownership + # claims, not the expected volatile observation timestamp or file inode. + claims = [] + for channel in ("beta", "release", "agent-table"): + path = root / channel / "supervisor" / "state.json" + state = json.loads(path.read_bytes()) + worker = state.get("worker") + if state.get("schema_version") != 1 or not isinstance(worker, dict) or not worker.get("process_run_id"): + raise AssertionError("actual worker native ownership evidence is absent") + claims.append( + { + "path": str(path), + "supervisor": {name: state.get(name) for name in ("schema_version", "operations", "selected_runtime_id", "launch_intent", "stopped", "stop_verified", "last_worker")}, + "worker": {name: worker.get(name) for name in ("pid", "start_token", "process_run_id", "deployment_generation", "runtime_id", "started_at", "stop_requested", "ready", "ready_path")}, + } + ) + configurations = tuple(root / name for name in ("compose.yml", "beta-secret", "release-secret", "agent-table-secret")) + return hashlib.sha256(canonical({"claims": claims, "configuration": [(str(path), path.stat().st_dev, path.stat().st_ino, _digest(path)) for path in configurations]})).hexdigest() + + +def _wait(predicate: Callable[[], bool], description: str, *, timeout: float = 90, process: subprocess.Popen | None = None) -> None: + deadline = time.monotonic() + timeout + while not predicate(): + if process is not None and process.poll() is not None: + raise AssertionError(description + ": the original process exited with " + str(process.returncode)) + if time.monotonic() >= deadline: + raise AssertionError(description + ": the bounded native barrier timed out") + time.sleep(0.05) + + +def _checkpoint_stopped(generation: Generation) -> None: + """The stopped fixture producer settles its own stores before inspection.""" + for directory in (generation.path("state"), generation.path("journal")): + for database in directory.glob("*.sqlite3"): + Chain.checkpoint(database) + + +def _no_target_child(target: Generation, *, running: bool) -> None: + assert not set(os.listdir(target.path("state"))) - {".supervisor-owner.lock", "logs"}, "prelaunch target created native consumer state" + assert not os.listdir(target.path("journal")), "prelaunch target admitted an operation" + if running: + state = json.loads(Chain.docker("inspect", "--format", "{{json .State}}", target.container)) + assert state["Running"] is True and state["Pid"] > 1 + pid = state["Pid"] + assert not Path(f"/proc/{pid}/task/{pid}/children").read_text().strip(), "prelaunch target has a child process" + + +def _native_refused_branch(chain: Chain, prior: Generation) -> Generation: + candidate = chain.prepare("D", prior) + assert candidate.intent is not None and candidate.proxy is not None + selected = FreshInitializationIntent(**json.loads(candidate.intent.read_bytes())) + bad = replace(selected, oauth_inode=selected.oauth_inode + 1) + # This is the real original admission producer. Its failure is retained in + # the prospective history; no native store/owner or host grant is fabricated. + try: + claim_fresh_initialization(candidate.path("state"), bad) + except ValueError as error: + assert str(error) == "retained OAuth root identity differs" + else: + raise AssertionError("the original producer accepted the mismatched OAuth inode") + assert not os.listdir(candidate.path("state")) and not os.listdir(candidate.path("journal")) + for identifier in (candidate.container, candidate.proxy): + state = json.loads(chain.docker("inspect", "--format", "{{json .State}}", identifier)) + assert state["Status"] == "created" and state["Pid"] == 0 and state["StartedAt"].startswith("0001-") + # Only these never-started physical objects are removed. Their original + # attempt/terminal boundaries and exact empty roots remain authoritative. + chain.docker("rm", candidate.proxy, candidate.container) + return candidate + + +def _unresolved_successor(chain: Chain, prior: Generation) -> Generation: + candidate = chain.prepare("C", prior) + host = chain.host(candidate) + chain.status(host, "ancestors-fenced-awaiting-manual-stop") + chain.stop(prior) + _checkpoint_stopped(prior) + chain.status(host, "awaiting-replacement") + chain.docker("start", candidate.container) + out, err = host.communicate(timeout=90) + assert host.returncode == 0, (out, err) + chain.ensure_proxy(candidate) + + def launched() -> bool: + try: + return read_fresh_status(candidate.path("state"), host=True)["status"] == "launched" and chain.child(candidate)["phase"] == "running" + except (OSError, ValueError, sqlite3.Error, TypeError, KeyError): + return False + + _wait(launched, "actual unresolved first launch") + # Withholding the authenticated readiness request leaves the genuine + # native outcome unresolved. No outcome file or SQLite row is rewritten. + store = ManagementOperationStore(candidate.path("journal")) + intent = OperationIntent(ManagedTarget(candidate.name, "management", MANAGEMENT_REPOSITORY), "management-service-self-update", (ActionParameter("source_sha", "a" * 40),), "native-retained-unknown") + admitted = store.admit(intent, chain.authority).record + assert admitted is not None + changed = store.transition(admitted.operation_id, 0, OperationPhase.NEEDS_ATTENTION, safe_error_category="UNKNOWN:fixture-effect-not-retried") + assert changed.record is not None + return candidate + + +def _check_lineage(chain: Chain, captures: tuple[CapturedGeneration, ...]) -> dict[str, str]: + assert chain.history_binding is not None + with ExitStack() as stack: + descriptors = [] + for captured in captures: + assert captured.original.owner is not None + stream = stack.enter_context(Path(captured.original.owner.path).open("rb")) + fcntl.flock(stream.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + descriptors.append(stream.fileno()) + history = stack.enter_context(observe(chain.history_binding)) + complete = derive_lineage(chain.history_binding, history, tuple(descriptors)) + assert not complete.unavailable, complete.unavailable + assert verify_lineage(complete, chain.history_binding, history, tuple(descriptors)) == complete + outcomes = {item.instance_id: item.outcome for item in complete.generations} + assert set(outcomes) == {"A", "B", "C", "D"} + assert outcomes["B"] == "succeeded" and outcomes["C"] in {"launched", "unknown"} and outcomes["D"] == "never-created" + # A true missing original is distinct from a captured unknown. Restore + # only this disposable original pathname before taking the baseline. + database = chain.generations[0].path("journal") / "management-operations.sqlite3" + hidden = database.with_name("temporarily-absent-original") + database.rename(hidden) + try: + missing = derive_lineage(chain.history_binding, history, tuple(descriptors)) + assert missing.unavailable and any(database.name in reason for reason in missing.unavailable) + try: + verify_lineage(complete, chain.history_binding, history, tuple(descriptors)) + except OriginalEvidenceUnavailable: + pass + else: + raise AssertionError("a saved positive lineage survived a missing original") + finally: + hidden.rename(database) + restored = derive_lineage(chain.history_binding, history, tuple(descriptors)) + assert not restored.unavailable and {item.instance_id: item.outcome for item in restored.generations} == outcomes + return outcomes + + +@dataclass(frozen=True) +class PreparedTarget: + generation: Generation + receipt: ManagementReadyReceipt + python: Path + consumer: Path + request_id: str + preparation_id: str + enrollment_id: str + published_port: int + + +def _target( + chain: Chain, + captures: tuple[CapturedGeneration, ...], + watchers: tuple[RecoveryWatcher, ...], + *, + authorization_id: str, + source_repository: Path, + source_token_file: Path, + source_sha: str, + uv: Path, + uv_version: str, +) -> PreparedTarget: + root = chain.root / "target" + root.mkdir(mode=0o700) + for name in ("state", "journal", "bootstrap"): + (root / name).mkdir(mode=0o700) + _write_private(root / "state" / ".supervisor-owner.lock", b"") + # Preserve the exact original OAuth setup, including its live grant table. + management = root / "bootstrap" / "management.yaml" + _write_private(management, (chain.root / "A" / "bootstrap" / "management.yaml").read_bytes()) + versions = root / "versions" + preparation = "recovery-" + uuid.uuid4().hex + configuration = ManagementRuntimeConfig("recovery-target", MANAGEMENT_REPOSITORY, source_repository, versions, Path(sys.executable), uv, uv_version, management_config=management, source_authority=_source_authority(source_token_file)) + reservation = ManagementVersionStager(configuration) + reservation._validate_boundaries() + reservation._reserve_root() + environment = dict(os.environ, UV_PYTHON_INSTALL_DIR=str(versions / ".python"), PYTHONDONTWRITEBYTECODE="1") + subprocess.run([str(uv), "python", "install", "3.12", "--no-bin"], check=True, env=environment, timeout=300) + python = Path(subprocess.check_output([str(uv), "python", "find", "--managed-python", "--system", "3.12"], env=environment, text=True, timeout=30).strip()).resolve(strict=True) + assert python.is_relative_to(versions.resolve()), "portable Python escaped the same-path version root" + staging = ManagementVersionStager(replace(configuration, python_executable=python)) + receipt = staging.prepare(ManagementPrepareIntent(source_sha, preparation)).receipt + assert staging.lookup(Path(receipt.environment_path).parent / "MANAGEMENT_READY.json") == receipt + bootstrap = BootstrapConfig( + "recovery-target", + Path("/management-child-state"), + Path("/management-oauth"), + Path("/management-operations"), + chain.origin + "/mcp", + receipt.management_runtime_id, + (), + crash_budget=0, + management_version_root=versions, + management_config=Path("/bootstrap/management.yaml"), + verification_timeout=600, + recovery_epoch_config=Path("/bootstrap/recovery-epoch.json"), + ) + runtime = runtime_for_receipt(bootstrap, receipt) + raw = { + "instance_id": bootstrap.instance_id, + "state_root": str(bootstrap.state_root), + "oauth_root": str(bootstrap.oauth_root), + "journal_root": str(bootstrap.journal_root), + "listener_url": bootstrap.listener_url, + "initial_runtime": receipt.management_runtime_id, + "runtimes": [{**asdict(runtime), "environment": dict(runtime.environment)}], + "crash_budget": 0, + "management_version_root": str(versions), + "management_config": str(bootstrap.management_config), + "verification_timeout": 600, + "recovery_epoch_config": "/bootstrap/recovery-epoch.json", + } + config = root / "bootstrap" / "config.json" + _write_private(config, json.dumps(raw, sort_keys=True, separators=(",", ":"), default=str).encode()) + BootstrapConfig.read(config) + consumer = root / "bootstrap" / "recovery-epoch.json" + _write_private(consumer, b"") + assert chain.history_binding is not None + mounts = [ + (root / "state", "/management-child-state", True), + (root / "journal", "/management-operations", True), + (chain.oauth, "/management-oauth", True), + (config, "/bootstrap/config.json", False), + (management, "/bootstrap/management.yaml", False), + (consumer, "/bootstrap/recovery-epoch.json", False), + (versions, str(versions), False), + (PEER_ROOT, "/management-recovery", False), + (Path(chain.history_binding.root) / "history.lock", "/management-recovery-history/history.lock", False), + ] + originals = tuple(sorted((item.original for item in captures), key=lambda item: item.generation_id)) + original_fences = [] + transfers = [] + for index, original in enumerate(originals): + assert original.owner is not None and original.namespace_pin is not None + destination = f"/management-recovery-originals/{index}/owner.lock" + mounts.append((Path(original.owner.path), destination, False)) + original_fences.append(OriginalFence(f"owner-{index}", destination, original.owner.device, original.owner.inode)) + transfers.append(TransferredResource(f"namespace-{index}", "network-namespace", original.namespace_pin.device, original.namespace_pin.inode)) + history = (Path(chain.history_binding.root) / "history.lock").stat() + original_fences.insert(0, OriginalFence("history", "/management-recovery-history/history.lock", history.st_dev, history.st_ino)) + for index, watcher in enumerate(sorted(watchers, key=lambda item: item.unit)): + transfers.append(TransferredResource(f"controller-{index}", "transferred-ex", watcher.lock.device, watcher.lock.inode)) + epoch = (EPOCH_ROOT / "epoch.lock").stat() + transfers.append(TransferredResource("epoch", "transferred-ex", epoch.st_dev, epoch.st_ino)) + name = "recovery-target-" + uuid.uuid4().hex + chain.names.append(name) + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + published_port = reservation.getsockname()[1] + if published_port == chain.port: + raise AssertionError("new target endpoint must differ from every old endpoint") + command = ["create", "--name", name, "--read-only", "--network", "bridge", "--publish", f"127.0.0.1:{published_port}:8080", "--env", "PYTHONDONTWRITEBYTECODE=1"] + for source, destination, writable in mounts: + command.extend(("--mount", f"type=bind,src={source},dst={destination}" + ("" if writable else ",readonly"))) + command.extend(("--entrypoint", "python", chain.image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json", "--recovery-epoch")) + identifier = chain.docker(*command) + generation = Generation("recovery-target", root, identifier, mounts) + socket_info, owner = PEER_ROOT.stat(), (root / "state" / ".supervisor-owner.lock").stat() + request_id = "request-" + uuid.uuid4().hex + selected = ConsumerConfiguration(1, request_id, authorization_id, identifier, 0, socket_info.st_dev, socket_info.st_ino, tuple(original_fences), TargetOwner("target-owner", owner.st_dev, owner.st_ino), tuple(transfers)) + with consumer.open("wb") as stream: + stream.write(canonical(asdict(selected))) + stream.flush() + os.fsync(stream.fileno()) + return PreparedTarget(generation, receipt, python, consumer, request_id, preparation, "enrollment-" + uuid.uuid4().hex, published_port) + + +def _install_authorization(chain: Chain, target: PreparedTarget, authorization_id: str, source_repository: Path, source_token_file: Path, source_sha: str, uv: Path, uv_version: str) -> None: + assert chain.history_binding is not None + socket_info, root, lock = PEER_ROOT.stat(), EPOCH_ROOT.stat(), (EPOCH_ROOT / "epoch.lock").stat() + authorization = EpochAuthorization( + AUTHORIZATION_SCHEMA, + chain.history_binding.deployment, + authorization_id, + target.request_id, + target.preparation_id, + target.generation.container, + target.enrollment_id, + chain.history_binding, + selected_file(target.consumer), + selected_file(Path(target.receipt.environment_path).parent / "MANAGEMENT_READY.json"), + str(source_repository), + str(source_token_file), + str(target.python), + str(uv), + uv_version, + source_sha, + socket_info.st_dev, + socket_info.st_ino, + ) + _write_private(AUTHORIZATION_FILE, canonical(asdict(authorization))) + inventory = EpochInventory( + 2, + str(EPOCH_ROOT), + root.st_dev, + root.st_ino, + lock.st_dev, + lock.st_ino, + authorization.deployment_id, + authorization_id, + (target.enrollment_id,), + target.request_id, + target.preparation_id, + target.generation.container, + source_identity().removeprefix("sha256:"), + _digest(target.generation.path("bootstrap") / "config.json"), + _digest(AUTHORIZATION_FILE), + ) + _write_private(EPOCH_ROOT / "inventory.json", canonical(asdict(inventory))) + + +def inspect_durable_result(root: Path, *, issuer_log: str, consumer_log: str) -> dict[str, object]: + """Check the native successful run's four original records and actual logs.""" + inventory = (root / "inventory.json").read_bytes() + inventory_sha = hashlib.sha256(inventory).hexdigest() + binding = None + previous = "" + digests = {} + for sequence, phase in enumerate(("reserved", "intent", "consumed", "receipt")): + encoded = (root / (phase + ".json")).read_bytes() + record = json.loads(encoded) + head_bytes = (root / (phase + "-head.json")).read_bytes() + head = json.loads(head_bytes) + assert canonical(record) == encoded and canonical(head) == head_bytes + assert record["schema_version"] == 1 and record["phase"] == phase and record["sequence"] == sequence + assert record["previous"] == previous and record["inventory_sha256"] == inventory_sha + assert head == {"schema_version": 1, "phase": phase, "record_sha256": hashlib.sha256(encoded).hexdigest(), "previous": previous, "inventory_sha256": inventory_sha} + if binding is None: + binding = record["binding"] + assert record["binding"] == binding + digests[phase] = hashlib.sha256(encoded).hexdigest() + previous = hashlib.sha256(head_bytes).hexdigest() + assert binding is not None + evidence = json.loads(binding["evidence_json"]) + assert hashlib.sha256(binding["evidence_json"].encode()).hexdigest() == binding["evidence_sha256"] + assert not evidence["lineage"]["unavailable"] and evidence["lineage"]["genesis"] is not None + assert evidence["inspection"]["unresolved_targets"] >= 1 and evidence["inspection"]["launch_authority"] is False + assert "One consumed recovery epoch grant send completed" in issuer_log + assert "Recovery custody acknowledged" in consumer_log + return { + "record_sha256": digests, + "evidence_sha256": binding["evidence_sha256"], + "inventory_sha256": inventory_sha, + "historical_keys": evidence["inspection"]["historical_keys"], + "unresolved_targets": evidence["inspection"]["unresolved_targets"], + "source_sha": evidence["target"]["receipt"]["source_sha"], + } + + +def validate_result(value: dict[str, object]) -> None: + """The CI oracle requires every native case and rejects partial acceptance.""" + assert value.get("schema") == RESULT_SCHEMA and value.get("status") == "passed" + scenario = value.get("scenario", "grant") + assert set(value.get("cases", ())) == required_cases(scenario) + assert value.get("no_child_started") is True and value.get("preserved_originals") is True and value.get("preserved_workers") is True + assert value.get("original_outcomes", {}).get("D") == "never-created" + assert value.get("original_outcomes", {}).get("C") in {"launched", "unknown"} + assert isinstance(value.get("durable"), dict) + if scenario == "grant": + assert set(value["durable"]["record_sha256"]) == {"reserved", "intent", "consumed", "receipt"} + else: + assert value["durable"]["phase"] == ("intent" if scenario == "kill-after-intent" else "consumed" if scenario == "kill-after-consume" else "receipt") + assert value["durable"]["consumer_grant_acknowledged"] is False + + +def _interrupted_issuer(scenario: str, issuer: subprocess.Popen, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: + """Signal real post-fsync publications; reject a race beyond the boundary.""" + phase = "intent" if scenario == "kill-after-intent" else "consumed" if scenario == "kill-after-consume" else "receipt" + selected_pid = issuer.pid if scenario != "lost-reply" else int(Chain.docker("inspect", "--format", "{{.State.Pid}}", target.generation.container)) + descriptor = os.pidfd_open(selected_pid, 0) + try: + _wait(lambda: f"Recovery epoch {phase} durably recorded" in issuer_log.read_text(), "actual durable " + phase + " publication", timeout=600, process=issuer) + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + finally: + os.close(descriptor) + if scenario == "lost-reply": + issuer.wait(timeout=120) + assert issuer.returncode == 2, "issuer accepted a dead consumer after durable consumption" + else: + issuer.wait(timeout=30) + assert issuer.returncode == -signal.SIGKILL + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "interrupted real peer lifetime ends", timeout=90) + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text(), "signal raced beyond the required pregrant boundary" + assert "One consumed recovery epoch grant send completed" not in issuer_log.read_text(), "signal raced beyond actual grant dispatch" + encoded = (EPOCH_ROOT / (phase + ".json")).read_bytes() + head = json.loads((EPOCH_ROOT / (phase + "-head.json")).read_bytes()) + assert head["phase"] == phase and head["record_sha256"] == hashlib.sha256(encoded).hexdigest() + if scenario == "kill-after-intent": + assert not (EPOCH_ROOT / "consumed.json").exists(), "intent interruption raced beyond consumption" + before = _snapshot((EPOCH_ROOT,)) + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == before, "an interrupted original epoch was repaired or reissued" + return {"phase": phase, "record_sha256": hashlib.sha256(encoded).hexdigest(), "consumer_grant_acknowledged": False, "grant_send_completed": False} + + +def _passed(result: dict[str, object], cases: list[str], preserved: str, worker_claims: str, worker_baseline: dict[str, object], worker_posts: dict[str, int]) -> None: + cases.extend(("originals-oauth-and-workers-preserved", "no-consumer-child-or-admission")) + result.update(status="passed", no_child_started=True, preserved_originals=True, preserved_workers=True, preserved_originals_sha256=preserved, preserved_worker_claims_sha256=worker_claims, worker_lifetimes=worker_baseline, worker_effect_posts=worker_posts) + validate_result(result) + logger.info("All {} native recovery ownership cases passed; no consumer child was launched", len(cases)) + + +def run_fixture(*, authorization_id: str, source_repository: Path, source_token_file: Path, source_sha: str, uv: Path, uv_version: str, output: Path, scenario: str = "grant") -> dict[str, object]: + required_cases(scenario) + if not re.fullmatch(r"[0-9a-f]{40}", source_sha) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", uv_version): + raise ValueError("an exact qualified source and installer version are required") + if any(not path.is_absolute() or path != path.resolve(strict=True) for path in (source_repository, source_token_file, uv)) or not output.is_absolute(): + raise ValueError("canonical absolute native fixture inputs are required") + for name in ("AUTOCODER_FRESH_TEST_BASE_IMAGE", "AUTOCODER_RECOVERY_CADDY_IMAGE"): + if re.fullmatch(r"sha256:[0-9a-f]{64}", os.environ.get(name, "")) is None: + raise ValueError("an exact already imported fixture image is required: " + name) + authorization = require_disposable_host(authorization_id) + # No resource is provisioned until the original marker selected this VM. + os.umask(0o077) + for directory in (FIXTURE_ROOT, EPOCH_ROOT, PEER_ROOT, AUTHORIZATION_ROOT): + if directory.exists() or directory.is_symlink(): + raise FileExistsError("recovery fixture refuses a previous/partial root: " + str(directory)) + _private_directory(FIXTURE_ROOT) + root = FIXTURE_ROOT / uuid.uuid4().hex + root.mkdir(mode=0o700) + for directory in (EPOCH_ROOT, PEER_ROOT, AUTHORIZATION_ROOT): + _private_directory(directory) + _write_private(EPOCH_ROOT / "epoch.lock", b"") + setup_logger(stream=sys.stdout, log_file=str(root / "fixture.log")) + cases: list[str] = [] + result: dict[str, object] = {"schema": RESULT_SCHEMA, "status": "failed", "scenario": scenario, "cases": cases, "authorization_id": authorization_id, "source_identity": source_identity(), "no_child_started": False, "preserved_originals": False, "preserved_workers": False} + chain = None + workers = None + stock = None + routes = None + watcher_fixtures: list[WatcherFixture] = [] + captures: list[CapturedGeneration] = [] + issuer = None + log_stream = None + target = None + try: + stock = InhibitedStockRoutes.install() + worker_root = root / "unrelated" + worker_root.mkdir(mode=0o700) + workers = WorkerFixture(worker_root, os.environ["AUTOCODER_FRESH_TEST_BASE_IMAGE"]) + worker_baseline, worker_posts = workers.workers(), workers.effect_posts() + worker_claims = _worker_claims(worker_root / "workers") + deployment_root = root / "deployment" + deployment_root.mkdir(mode=0o700) + chain = Chain(deployment_root, original_genesis=True) + original = chain.prepare("A") + assert chain.genesis is not None and chain.history_binding is not None + cases.append("affirmative-original-genesis") + chain.docker("start", original.container) + _wait(lambda: (original.path("state") / "original-genesis-first-launch.json").is_file(), "actual original first-launch boundary") + chain.authorize() + chain.ready(original, fresh=False) + chain.prove_child(original) + chain.seed(original) + captures.append(capture_generation(chain, original)) + fixture, watcher = start_watcher(original, captures[-1].original, first=True) + watcher_fixtures.append(fixture) + watchers = [watcher] + second = chain.prepare("B", original) + chain.start_cutover(original, second) + _checkpoint_stopped(original) + chain.seed(second) + captures.append(capture_generation(chain, second)) + fixture, watcher = start_watcher(second, captures[-1].original) + watcher_fixtures.append(fixture) + watchers.append(watcher) + cases.append("native-successor-ready") + _native_refused_branch(chain, second) + cases.append("native-failed-branch-retained") + third = _unresolved_successor(chain, second) + captures.append(capture_generation(chain, third)) + fixture, watcher = start_watcher(third, captures[-1].original) + watcher_fixtures.append(fixture) + watchers.append(watcher) + chain.stop(third) + _checkpoint_stopped(third) + for captured in captures: + assert captured.original.namespace_pin is not None + assert os.readlink(f"/proc/self/fd/{captured.namespace_fd}") == captured.original.namespace_pin.namespace + result["original_outcomes"] = _check_lineage(chain, tuple(captures)) + cases.extend(("native-unresolved-successor-retained", "missing-original-refused")) + target = _target(chain, tuple(captures), tuple(watchers), authorization_id=authorization_id, source_repository=source_repository, source_token_file=source_token_file, source_sha=source_sha, uv=uv, uv_version=uv_version) + routes = DisposableSystemdRoutes.install(fixture_id="ac-test-pr2829-" + uuid.uuid4().hex, manager_id=third.container) + routes.assert_inhibited() + stock.assert_inhibited() + selected_target = target_from_container(target.generation.container, bootstrap_config=target.generation.path("bootstrap") / "config.json", management_config=target.generation.path("bootstrap") / "management.yaml", published_port=target.published_port) + enroll_recovery_scope(tuple(captures), selected_target, tuple(watchers), deployment_id=chain.history_binding.deployment, enrollment_id=target.enrollment_id, authorization=authorization) + _install_authorization(chain, target, authorization_id, source_repository, source_token_file, source_sha, uv, uv_version) + retained_roots = (Path(chain.history_binding.root), chain.oauth, *(path for generation in chain.generations for path in (generation.path("state"), generation.path("journal"), generation.path("bootstrap")))) + Chain.checkpoint(chain.oauth / "oauth.sqlite3") + preserved = _snapshot(retained_roots) + worker_before = (worker_baseline, worker_posts, worker_claims) + workers.preserved(*worker_before) + issuer_log = root / "issuer.log" + log_stream = issuer_log.open("xb") + issuer = subprocess.Popen([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], stdout=log_stream, stderr=subprocess.STDOUT, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + _wait(lambda: PEER_SOCKET.exists() and "Fixed recovery issuer endpoint is listening" in issuer_log.read_text(), "actual issuer listening", process=issuer) + contender = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert contender.returncode == 2 and not (EPOCH_ROOT / "reserved.json").exists(), "a second issuer acquired the same current epoch" + cases.append("independent-issuer-contender-refused") + # A real SIGSTOP barrier lets the exact target execute its installed + # interpreter probe before T takes its current process observations. + issuer.send_signal(signal.SIGSTOP) + _wait(lambda: Path(f"/proc/{issuer.pid}/stat").read_text().rsplit(")", 1)[1].split()[0] == "T", "issuer kernel stop barrier", process=issuer) + chain.docker("start", target.generation.container) + probe = json.loads(chain.docker("exec", target.generation.container, target.receipt.interpreter_path, "-I", "-B", "-m", "auto_coder.build_provenance", timeout=60).splitlines()[-1]) + assert probe["source_revision"]["available"] is True and probe["source_revision"]["value"] == source_sha + assert probe["distribution_version"]["available"] is True and probe["distribution_version"]["value"] == target.receipt.distribution_version + _no_target_child(target.generation, running=True) + cases.append("genuine-installed-ready-in-target") + issuer.send_signal(signal.SIGCONT) + consumer_log = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" + result["source_proof"] = {name: getattr(target.receipt, name) for name in ("source_sha", "tested_source_ref", "tested_source_object", "test_run_id", "test_run_attempt", "lock_sha256", "distribution_version", "receipt_identity")} + if scenario != "grant": + result["durable"] = _interrupted_issuer(scenario, issuer, issuer_log, consumer_log, target) + cases.extend(("native-" + scenario, "interrupted-epoch-replay-refused")) + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) + _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) + return result + _wait(lambda: (EPOCH_ROOT / "receipt-head.json").is_file() and consumer_log.is_file() and "Recovery custody acknowledged" in consumer_log.read_text(), "native grant and consumer custody", timeout=600, process=issuer) + result["durable"] = inspect_durable_result(EPOCH_ROOT, issuer_log=issuer_log.read_text(), consumer_log=consumer_log.read_text()) + cases.append("complete-native-epoch-granted") + for captured in captures: + assert captured.original.owner is not None + with Path(captured.original.owner.path).open("rb") as stream: + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + pass + else: + raise AssertionError("old original owner was not excluded by actual SH custody") + for watcher in watchers: + with Path(watcher.lock.path).open("rb") as stream: + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + pass + else: + raise AssertionError("actual watcher EX custody was released") + cases.append("old-owner-and-controller-contenders-refused") + _no_target_child(target.generation, running=True) + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) + durable_before = _snapshot((EPOCH_ROOT,)) + issuer.kill() + issuer.wait(timeout=30) + _wait(lambda: chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "actual issuer death invalidates consumer custody") + assert chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + _no_target_child(target.generation, running=False) + cases.append("issuer-death-ended-current-custody") + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == durable_before, "a consumed epoch was reissued or repaired" + cases.append("consumed-epoch-replay-refused") + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) + _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) + return result + except BaseException as error: + result["failure_type"] = type(error).__name__ + logger.exception("Native recovery ownership fixture failed") + raise + finally: + diagnostics = [] + for path in (root / "issuer.log", root / "fixture.log"): + if path.is_file(): + diagnostics.extend(diagnostic_log_lines(path.read_text(errors="replace"))) + if target is not None: + path = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" + if path.is_file(): + diagnostics.extend(diagnostic_log_lines(path.read_text(errors="replace"))) + result["diagnostics"] = diagnostics[-120:] + output.parent.mkdir(parents=True, exist_ok=True) + _write_private(output, canonical(result)) + if issuer is not None and issuer.poll() is None: + issuer.send_signal(signal.SIGCONT) + issuer.terminate() + try: + issuer.wait(timeout=15) + except subprocess.TimeoutExpired: + issuer.kill() + issuer.wait(timeout=15) + if log_stream is not None: + log_stream.close() + if chain is not None: + chain.close() + for captured in reversed(captures): + captured.close() + for watcher_fixture in reversed(watcher_fixtures): + watcher_fixture.cleanup() + if routes is not None: + routes.cleanup() + if stock is not None: + stock.cleanup() + if workers is not None: + workers.close() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--authorization-id", required=True) + parser.add_argument("--source-repository", type=Path, required=True) + parser.add_argument("--source-token-file", type=Path, required=True) + parser.add_argument("--source-sha", required=True) + parser.add_argument("--uv", type=Path, required=True) + parser.add_argument("--uv-version", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--scenario", choices=SCENARIOS, default="grant") + args = parser.parse_args(argv) + run_fixture(**vars(args)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/auto_coder/management/bootstrap_supervisor.py b/src/auto_coder/management/bootstrap_supervisor.py index 7970b21d3..f5783b86c 100644 --- a/src/auto_coder/management/bootstrap_supervisor.py +++ b/src/auto_coder/management/bootstrap_supervisor.py @@ -205,6 +205,8 @@ class BootstrapConfig: new_instance_initialization: bool = False new_docker_network: str | None = None new_docker_host_port: int | None = None + original_genesis: Path | None = None + recovery_epoch_config: Path | None = None @classmethod def read(cls, path: Path) -> "BootstrapConfig": @@ -232,6 +234,8 @@ def read(cls, path: Path) -> "BootstrapConfig": "new_instance_initialization", "new_docker_network", "new_docker_host_port", + "original_genesis", + "recovery_epoch_config", } if set(raw) - allowed: raise ValueError("unknown bootstrap configuration field") @@ -273,6 +277,8 @@ def read(cls, path: Path) -> "BootstrapConfig": raw.get("new_instance_initialization", False), raw.get("new_docker_network"), raw.get("new_docker_host_port"), + Path(raw["original_genesis"]) if raw.get("original_genesis") else None, + Path(raw["recovery_epoch_config"]) if raw.get("recovery_epoch_config") else None, ) config.validate() return config @@ -295,6 +301,12 @@ def validate(self) -> None: raise ValueError("invalid management self-update bootstrap profile") if type(self.new_instance_initialization) is not bool: raise ValueError("NEW initialization selector must be a boolean") + if self.original_genesis is not None: + if self.original_genesis != Path("/bootstrap/original-genesis.json") or self.new_instance_initialization or self.fresh_guard_path is not None or self.fresh_prior_lock_path is not None or self.archived_exclusions or self.fresh_runtime_binding is not None: + raise ValueError("original genesis requires the explicitly selected ordinary bootstrap profile") + if self.recovery_epoch_config is not None: + if self.recovery_epoch_config != Path("/bootstrap/recovery-epoch.json") or self.original_genesis is not None or self.new_instance_initialization or self.fresh_guard_path is not None or self.fresh_prior_lock_path is not None or self.fresh_runtime_binding is not None: + raise ValueError("recovery epoch requires its separate explicitly selected bootstrap profile") if (self.new_docker_network is None) != (self.new_docker_host_port is None): raise ValueError("NEW Docker loopback network and port must be selected together") if self.new_docker_network is not None: @@ -369,6 +381,8 @@ class ManagementBootstrapSupervisor: def __init__(self, config: BootstrapConfig, incarnation: str | None = None, bootstrap_config_path: Path | None = None, *, fresh_initialization: bool = False, new_initialization_request: Path | None = None, source_authority: TestedSourcePublisher | None = None) -> None: config.validate() + if config.recovery_epoch_config is not None: + raise ValueError("recovery custody cannot enter the ordinary child supervisor") if bool(new_initialization_request) != config.new_instance_initialization or (fresh_initialization and new_initialization_request is not None): raise ValueError("NEW bootstrap must select only its dedicated first-launch path") from .worker_network_client import configured_worker_bridge_environment @@ -401,6 +415,31 @@ def _owner_file(self) -> Iterator[IO[bytes]]: unenrolled Supervisor profiles keep their existing opening behavior. """ path = self.config.state_root / ".supervisor-owner.lock" + if self.config.original_genesis is not None: + from .original_attempts import history_lock, identity + from .original_genesis import HISTORY_MOUNT, LAUNCH_NAME, NATIVE_NAME, validate_history_genesis + + with history_lock(HISTORY_MOUNT, exclusive=False): + original = validate_history_genesis(HISTORY_MOUNT, mounted_genesis=self.config.original_genesis) + original_state = original.root("state") + if identity(self.config.state_root, directory=True) != (original_state.device, original_state.inode): + raise PermissionError("original genesis owner root differs") + # An interrupted first owner cannot be retried on an empty-looking + # root. Reconstruction requires both original native boundaries. + recorded = all((self.config.state_root / name).is_file() for name in (NATIVE_NAME, LAUNCH_NAME)) + flags = os.O_RDWR | os.O_NOFOLLOW | (0 if recorded else os.O_CREAT | os.O_EXCL) + descriptor = os.open(path, flags, 0o600) + with os.fdopen(descriptor, "a+b") as owner: + info = os.fstat(owner.fileno()) + if (info.st_dev, info.st_ino) != identity(path): + raise PermissionError("original genesis owner lock differs") + if not recorded: + os.fsync(owner.fileno()) + from .original_attempts import _sync + + _sync(self.config.state_root) + yield owner + return if self.new_initialization_request is not None: # O_EXCL is a durable fail-closed lifetime gate. A crashed or # partially published NEW owner cannot acquire a second owner. @@ -830,6 +869,9 @@ def _pristine_initial_state(self) -> bool: def run(self) -> int: """Remain alive as PID 1 and reap only the configured child.""" + if self.config.original_genesis is not None and (not self.config.state_root.is_dir() or self.config.state_root.is_symlink()): + logger.error("Original genesis state root is unavailable; refusing reconstruction") + return 1 if self.config.new_instance_initialization and (self.new_initialization_request is None or not self.config.state_root.is_dir() or self.config.state_root.is_symlink()): logger.error("NEW initialization requires an explicit original request and protected state root") return 1 @@ -897,6 +939,15 @@ def _run_owned(self) -> int: from .self_update_handoff import SelfUpdateHandoffConsumer owner_start_token = self._await_owner_start_token() + original_first_launch = None + if self.config.original_genesis is not None: + from .original_genesis import prepare_original_genesis + + try: + original_first_launch = prepare_original_genesis(self, owner_start_token) + except (OSError, ValueError, TypeError, KeyError, sqlite3.Error): + logger.exception("Original ordinary genesis is unavailable; preserving originals without launch") + return 1 if self.new_initialization_request is not None: if not self._start_new_initialization(): return 1 @@ -980,7 +1031,7 @@ def _run_owned(self) -> int: self.store.require_existing = True consumer = SelfUpdateHandoffConsumer(self) if self.config.management_config is not None else None record = self.store.load() - initial_state_pristine = self._pristine_initial_state() if record.child_generation == 0 else False + initial_state_pristine = original_first_launch is not None or (self._pristine_initial_state() if record.child_generation == 0 else False) if consumer is not None and record.child_generation == 0 and not consumer.store.path.exists() and not consumer.store.path.is_symlink(): try: consumer.store.active() @@ -1006,7 +1057,8 @@ def _run_owned(self) -> int: if not initial_state_pristine or self.store.has_unresolved_effects(): self.store.reserve(record, replace(record, phase=ChildPhase.UNKNOWN, exit_cause="unresolved_effect")) else: - self.start_selected(f"bootstrap-{uuid.uuid4()}", str(uuid.uuid4()), self.config.initial_runtime) + operation, attempt = original_first_launch or (f"bootstrap-{uuid.uuid4()}", str(uuid.uuid4())) + self.start_selected(operation, attempt, self.config.initial_runtime) if self.config.restart_after_clean_shutdown and record.child_generation > 0 and not self._container_shutdown_requested: from .clean_restart import restart_verified_stop @@ -1476,9 +1528,16 @@ def main() -> None: parser.add_argument("--config", required=True, type=Path) parser.add_argument("--fresh-initialization", action="store_true") parser.add_argument("--new-initialization", action="store_true") + parser.add_argument("--recovery-epoch", action="store_true") parser.add_argument("--request-file", type=Path) args = parser.parse_args() config = BootstrapConfig.read(args.config) + if config.recovery_epoch_config is not None or args.recovery_epoch: + if not args.recovery_epoch or config.recovery_epoch_config is None or args.new_initialization or args.fresh_initialization or args.request_file is not None: + parser.exit(1, "Recovery epoch requires its distinct fixed bootstrap profile\n") + from .recovery_epoch_bootstrap import run_recovery_epoch_bootstrap + + raise SystemExit(run_recovery_epoch_bootstrap(config)) if config.new_instance_initialization: if not args.new_initialization or args.fresh_initialization or args.request_file is None: parser.exit(1, "NEW initialization requires a separately admitted first-launch controller\n") @@ -1492,6 +1551,13 @@ def main() -> None: parser.exit(1, f"NEW first-launch original unavailable: {error}\n") elif args.new_initialization or args.request_file is not None: parser.exit(1, "NEW initialization flags require a selected NEW profile\n") + if config.original_genesis is not None: + from .original_genesis import validate_bootstrap_genesis + + try: + validate_bootstrap_genesis(config, args.config) + except (OSError, ValueError, TypeError, KeyError) as error: + parser.exit(1, f"Ordinary genesis original unavailable: {error}\n") setup_logger(stream=sys.stdout, log_file=str(config.state_root / "logs" / "management-bootstrap.log")) raise SystemExit(ManagementBootstrapSupervisor(config, bootstrap_config_path=args.config, fresh_initialization=args.fresh_initialization, new_initialization_request=args.request_file).run()) diff --git a/src/auto_coder/management/containment_observation.py b/src/auto_coder/management/containment_observation.py index b0e49c283..43cd82826 100644 --- a/src/auto_coder/management/containment_observation.py +++ b/src/auto_coder/management/containment_observation.py @@ -174,6 +174,22 @@ def _docker_read(arguments: tuple[str, ...]) -> object: def _docker(scope: ProposedScope) -> tuple[DockerObservation, ...]: + return _docker_selected((scope.management_id,), (scope.proxy_id,), all_bind_sources=False) + + +def observe_recovery_docker(management_ids: tuple[str, ...], proxy_ids: tuple[str, ...]) -> tuple[DockerObservation, ...]: + """Enumerate the whole engine against the explicitly selected recovery union. + + This only discovers candidates. The recovery observer separately validates + the protected selection, every object profile and all installed routes. + """ + return _docker_selected(management_ids, proxy_ids, all_bind_sources=True) + + +def _docker_selected(management_ids: tuple[str, ...], proxy_ids: tuple[str, ...], *, all_bind_sources: bool) -> tuple[DockerObservation, ...]: + selected = management_ids + proxy_ids + if not management_ids or not proxy_ids or len(selected) > 64 or len(set(selected)) != len(selected) or any(not isinstance(item, str) or not re.fullmatch(r"[0-9a-f]{64}", item) for item in selected): + raise ValueError("selected Docker identities are absent, duplicate or ambiguous") # A remote context cannot supply this host's kernel/Docker correlation. if os.environ.get("DOCKER_HOST") or os.environ.get("DOCKER_CONTEXT"): raise ValueError("Docker endpoint override is unsupported for local observation") @@ -186,7 +202,7 @@ def _docker(scope: ProposedScope) -> tuple[DockerObservation, ...]: # Enumerate the engine, not an operator/label-filtered selection. Unknown # objects remain visible; enumeration does not establish their enrollment. containers = _docker_ids() - if not {scope.management_id, scope.proxy_id} <= set(containers): + if not set(selected) <= set(containers): raise ValueError("Selected Docker objects are absent from the complete engine inventory") observations = [] inspected: dict[str, dict[str, object]] = {} @@ -219,12 +235,15 @@ def _docker(scope: ProposedScope) -> tuple[DockerObservation, ...]: # its exact images, an installed management command or its network namespace. # This is candidate discovery, not proof that all ordinary controllers are # known; the separate installed-route inventory is still required. - management_image = inspected[scope.management_id]["Image"] - proxy_image = inspected[scope.proxy_id]["Image"] - management_mounts = inspected[scope.management_id]["Mounts"] - if not isinstance(management_mounts, list): - raise ValueError("Selected management mount inventory is unavailable") - management_sources = tuple(source for mount in management_mounts if isinstance(mount, dict) and mount.get("Type") == "bind" and mount.get("Destination") in _MANAGEMENT_MOUNT_DESTINATIONS and isinstance(source := mount.get("Source"), str) and os.path.isabs(source)) + management_images = {inspected[identifier]["Image"] for identifier in management_ids} + proxy_images = {inspected[identifier]["Image"] for identifier in proxy_ids} + management_mounts: list[object] = [] + for identifier in management_ids: + mounts = inspected[identifier]["Mounts"] + if not isinstance(mounts, list): + raise ValueError("Selected management mount inventory is unavailable") + management_mounts.extend(mounts) + management_sources = tuple(source for mount in management_mounts if isinstance(mount, dict) and mount.get("Type") == "bind" and (all_bind_sources or mount.get("Destination") in _MANAGEMENT_MOUNT_DESTINATIONS) and isinstance(source := mount.get("Source"), str) and os.path.isabs(source)) def overlapping_management_source(candidate_mounts: object) -> bool: if not isinstance(candidate_mounts, list): @@ -259,19 +278,19 @@ def overlapping_management_source(candidate_mounts: object) -> bool: raise ValueError("Docker network mode is ambiguous") arguments = tuple((entrypoint or []) + (command or [])) roles = [] - if item.container_id == scope.management_id: + if item.container_id in management_ids: roles.append("selected-management") - elif item.image_id == management_image: + elif item.image_id in management_images: roles.append("same-management-image") - if item.container_id == scope.proxy_id: + if item.container_id in proxy_ids: roles.append("selected-proxy") - elif item.image_id == proxy_image: + elif item.image_id in proxy_images: roles.append("same-proxy-image") if "auto_coder.management.bootstrap_supervisor" in arguments or ("auto_coder.cli" in arguments and "management" in arguments and "serve" in arguments): roles.append("management-command") - if network_mode == "container:" + scope.management_id: + if network_mode in {"container:" + identifier for identifier in management_ids}: roles.append("selected-management-network") - if item.container_id not in (scope.management_id, scope.proxy_id) and overlapping_management_source(obj["Mounts"]): + if item.container_id not in selected and overlapping_management_source(obj["Mounts"]): roles.append("selected-management-mount-source") if roles or "ExecIDs" not in obj: classified.append(replace(item, management_route_roles=tuple(roles))) diff --git a/src/auto_coder/management/containment_route_manifest.py b/src/auto_coder/management/containment_route_manifest.py index 1753f8289..445096608 100644 --- a/src/auto_coder/management/containment_route_manifest.py +++ b/src/auto_coder/management/containment_route_manifest.py @@ -26,6 +26,7 @@ ENROLLMENT_ROOT, EnrollmentObservation, EnrollmentUnavailable, + OriginalFile, _approved_directory, _file_identity, _protected, @@ -113,6 +114,34 @@ class RouteManifestUnavailable(ValueError): """An approved, complete and currently inhibited route set is unavailable.""" +@dataclass(frozen=True) +class RecoveryRouteScope: + """The protected recovery selection, without a historical or grant claim.""" + + deployment_id: str + enrollment_id: str + authorization_id: str + profile: str + container_ids: tuple[str, ...] + original_files: tuple[OriginalFile, ...] + + +@dataclass(frozen=True) +class RecoveryRouteSelection: + enrollment: RecoveryRouteScope + enrollment_sha256: str + + +def _route_container_ids(approval: EnrollmentObservation | RecoveryRouteSelection) -> tuple[str, ...]: + if isinstance(approval, RecoveryRouteSelection): + scope = approval.enrollment + selected = scope.container_ids + if scope.profile != "ordinary-recovery-v1" or not 3 <= len(selected) <= 64 or selected != tuple(sorted(set(selected))) or any(not _CONTAINER.fullmatch(item) for item in selected): + raise RouteManifestUnavailable("recovery route selection differs") + return selected + return approval.enrollment.management_id, approval.enrollment.proxy_id + + @dataclass(frozen=True) class SystemdRoute: unit: str @@ -282,9 +311,10 @@ def parse(cls, value: object) -> ApprovedRouteManifest: bindings: tuple[DeploymentRootBinding, ...] = () user_digest = "" user_routes: tuple[UserSystemdRoute, ...] = () - elif value.get("schema") == "approved-management-ordinary-routes-v2" and set(value) == set(cls.__dataclass_fields__): + elif value.get("schema") in ("approved-management-ordinary-routes-v2", "approved-management-recovery-routes-v1") and set(value) == set(cls.__dataclass_fields__): raw_bindings = value["deployment_roots"] - if not isinstance(raw_bindings, list) or not 1 <= len(raw_bindings) <= 128: + limit = 1024 if value["schema"] == "approved-management-recovery-routes-v1" else 128 + if not isinstance(raw_bindings, list) or not 1 <= len(raw_bindings) <= limit: raise RouteManifestUnavailable("approved deployment root inventory is absent or oversized") bindings = tuple(DeploymentRootBinding.parse(item) for item in raw_bindings) binding_keys = tuple((item.container_id, item.destination) for item in bindings) @@ -307,7 +337,8 @@ def parse(cls, value: object) -> ApprovedRouteManifest: for name in ("deployment_id", "enrollment_id", "authorization_id"): if not isinstance(value[name], str) or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", value[name]): raise RouteManifestUnavailable("approved ordinary-route deployment differs") - if value["profile"] not in ("ordinary-v1", "quarantined-original-v1"): + profiles = ("ordinary-recovery-v1",) if value["schema"] == "approved-management-recovery-routes-v1" else ("ordinary-v1", "quarantined-original-v1") + if value["profile"] not in profiles: raise RouteManifestUnavailable("approved ordinary-route profile differs") if not isinstance(value["enrollment_sha256"], str) or not _HASH.fullmatch(value["enrollment_sha256"]): raise RouteManifestUnavailable("approved ordinary-route enrollment binding differs") @@ -626,14 +657,20 @@ def _user_controllers(systemd: tuple[SystemdRouteFact, ...], cues: tuple[str, .. return inventory, tuple(sorted(units, key=lambda item: (item.uid, item.fact.unit))) -def _deployment_root_bindings(approval: EnrollmentObservation, *, trusted_uid: int = 0) -> tuple[DeploymentRootBinding, ...]: +def _deployment_root_bindings(approval: EnrollmentObservation | RecoveryRouteSelection, *, trusted_uid: int = 0) -> tuple[DeploymentRootBinding, ...]: """Derive scope roots from the exact enrolled pair, never a caller root list.""" + return observe_selected_bind_roots(_route_container_ids(approval), trusted_uid=trusted_uid) + + +def observe_selected_bind_roots(selected: tuple[str, ...], *, trusted_uid: int = 0) -> tuple[DeploymentRootBinding, ...]: + """Read actual selected-object bind identities; this does not enroll them.""" if os.environ.get("DOCKER_HOST") or os.environ.get("DOCKER_CONTEXT"): raise RouteManifestUnavailable("remote Docker root discovery is unsupported") - selected = (approval.enrollment.management_id, approval.enrollment.proxy_id) + if not 1 <= len(selected) <= 64 or len(set(selected)) != len(selected) or any(not isinstance(item, str) or not _CONTAINER.fullmatch(item) for item in selected): + raise RouteManifestUnavailable("selected bind-root Docker identities differ") raw = _command("docker", "inspect", "--type", "container", *selected, max_bytes=8 * _MAX_MANIFEST_BYTES) decoded = json.loads(raw, object_pairs_hook=unique_pairs) - if not isinstance(decoded, list) or len(decoded) != 2 or {item.get("Id") for item in decoded if isinstance(item, dict)} != set(selected): + if not isinstance(decoded, list) or len(decoded) != len(selected) or {item.get("Id") for item in decoded if isinstance(item, dict)} != set(selected): raise RouteManifestUnavailable("deployment root Docker scope differs") result = [] for container in decoded: @@ -821,14 +858,14 @@ def _local_script(path: Path, *, expected_uid: int) -> tuple[str, int, int]: return hashlib.sha256(data).hexdigest(), after.st_dev, after.st_ino -def _deployment_cues(approval: EnrollmentObservation, bindings: tuple[DeploymentRootBinding, ...]) -> tuple[str, ...]: - return tuple(sorted({item.source for item in bindings} | {item.path for item in approval.enrollment.original_files} | {approval.enrollment.management_id, approval.enrollment.proxy_id} | set(CRON_ROUTE_CUES))) +def _deployment_cues(approval: EnrollmentObservation | RecoveryRouteSelection, bindings: tuple[DeploymentRootBinding, ...]) -> tuple[str, ...]: + return tuple(sorted({item.source for item in bindings} | {item.path for item in approval.enrollment.original_files} | set(_route_container_ids(approval)) | set(CRON_ROUTE_CUES))) -def _deployment_modules(approval: EnrollmentObservation) -> tuple[tuple[str, str], ...]: +def _deployment_modules(approval: EnrollmentObservation | RecoveryRouteSelection) -> tuple[tuple[str, str], ...]: # This known original controller is supported only through its separately # approved source binding and isolated Python command, not arbitrary -m. - return tuple(("auto_coder.management.proxy_watch_cli", item.path) for item in approval.enrollment.original_files if getattr(item, "purpose", None) == "watcher-source") + return tuple(sorted({("auto_coder.management.proxy_watch_cli", item.path) for item in approval.enrollment.original_files if getattr(item, "purpose", None) == "watcher-source"})) def _validate_user_scope(manifest: ApprovedRouteManifest, inventory: UserControllerInventory, facts: tuple[UserSystemdRouteFact, ...]) -> None: @@ -857,7 +894,7 @@ def _validate_user_scope(manifest: ApprovedRouteManifest, inventory: UserControl def _observe_approved_route_manifest( - approval: EnrollmentObservation, + approval: EnrollmentObservation | RecoveryRouteSelection, docker: tuple[object, ...], *, root: Path, @@ -883,7 +920,9 @@ def _observe_approved_route_manifest( ids = tuple(sorted(getattr(item, "container_id", "") for item in docker)) if ids != manifest.docker_container_ids: raise RouteManifestUnavailable("whole-engine Docker inventory differs from approved routes") - is_closed_version = manifest.schema == "approved-management-ordinary-routes-v2" + is_closed_version = manifest.schema in ("approved-management-ordinary-routes-v2", "approved-management-recovery-routes-v1") + if isinstance(approval, RecoveryRouteSelection) != (manifest.schema == "approved-management-recovery-routes-v1"): + raise RouteManifestUnavailable("route manifest and selection profile differ") bindings = _deployment_root_bindings(approval, trusted_uid=expected_uid) if is_closed_version else () if bindings != manifest.deployment_roots: raise RouteManifestUnavailable("original deployment root/config binding differs from approved routes") @@ -945,13 +984,23 @@ def observe_approved_route_manifest(approval: EnrollmentObservation, docker: tup return _observe_approved_route_manifest(approval, docker, root=ENROLLMENT_ROOT, expected_uid=0) -def propose_unapproved_route_draft(approval: EnrollmentObservation, docker: tuple[object, ...], *, closed_profile: bool = False) -> dict[str, object]: +def observe_recovery_route_manifest(approval: RecoveryRouteSelection, docker: tuple[object, ...]) -> RouteManifestObservation: + """Read only the fixed recovery approval and the entire real route scope.""" + from .recovery_containment_model import RECOVERY_ENROLLMENT_ROOT + + return _observe_approved_route_manifest(approval, docker, root=RECOVERY_ENROLLMENT_ROOT, expected_uid=0) + + +def propose_unapproved_route_draft(approval: EnrollmentObservation | RecoveryRouteSelection, docker: tuple[object, ...], *, closed_profile: bool = False) -> dict[str, object]: """Provide inspectable values for a human review; never attest completeness. All exclusions and absent local/remote sources are unreviewed. This draft cannot be installed as usable approval until a trusted root explicitly audits and replaces the false completeness declaration. """ + recovery = isinstance(approval, RecoveryRouteSelection) + if recovery and not closed_profile: + raise RouteManifestUnavailable("recovery requires complete current user/root observations") bindings = _deployment_root_bindings(approval) if closed_profile else () cues = _deployment_cues(approval, bindings) if closed_profile else () modules = _deployment_modules(approval) if closed_profile else () @@ -960,7 +1009,7 @@ def propose_unapproved_route_draft(approval: EnrollmentObservation, docker: tupl cron = _cron_catalog(roots=_CRON_ROOTS, extra_cues=cues, inspect_local_inputs=closed_profile, module_sources=modules, systemd_facts=systemd) users, user_units = _user_controllers(systemd, cues, modules) if closed_profile else (None, ()) return { - "schema": "approved-management-ordinary-routes-v2" if closed_profile else "approved-management-ordinary-routes-v1", + "schema": "approved-management-recovery-routes-v1" if recovery else "approved-management-ordinary-routes-v2" if closed_profile else "approved-management-ordinary-routes-v1", "deployment_id": approval.enrollment.deployment_id, "enrollment_id": approval.enrollment.enrollment_id, "authorization_id": approval.enrollment.authorization_id, diff --git a/src/auto_coder/management/exclusion_host.py b/src/auto_coder/management/exclusion_host.py index 679a5409e..22588ad56 100644 --- a/src/auto_coder/management/exclusion_host.py +++ b/src/auto_coder/management/exclusion_host.py @@ -73,6 +73,21 @@ def inherited_support(mounts: list[object]) -> tuple[tuple[ArchivedExclusion, .. if raw.get("fresh_prior_lock_path") is None: if archived: raise ValueError("archives lack fresh predecessor provenance") + if raw.get("original_genesis") is not None: + from .original_genesis import GENESIS_NAME, HISTORY_MOUNT, read_genesis, validate_history_genesis + + if raw["original_genesis"] != "/bootstrap/" + GENESIS_NAME: + raise ValueError("ordinary original genesis path differs") + original = read_genesis(bootstrap.parent / GENESIS_NAME) + if validate_history_genesis(Path(original.binding.root)) != original or original.instance_id != raw.get("instance_id") or original.file("bootstrap").sha256 != hashlib.sha256(content).hexdigest(): + raise ValueError("ordinary original genesis authority differs") + for role in ("state", "journal", "oauth", "history"): + expected = original.root(role) + selected = _mount(mounts, expected.container_path, rw=role != "history") + current = selected.lstat() + if str(selected) != expected.host_path or (current.st_dev, current.st_ino) != (expected.device, expected.inode): + raise ValueError("ordinary original genesis mounted root differs") + return (), {str(HISTORY_MOUNT)} return (), set() if raw.get("state_root") != "/management-child-state" or raw.get("fresh_prior_lock_path") != "/management-prior-owner.lock": raise ValueError("predecessor fresh paths differ") @@ -119,6 +134,16 @@ def required_exclusions(prior: PriorContainer) -> tuple[ArchivedExclusion, ...]: def validate_successor(prior: PriorContainer, intent: FreshInitializationIntent, bootstrap_path: Path | None) -> tuple[ArchivedExclusion, ...]: required = required_exclusions(prior) prior_config = json.loads(protected_bytes(Path(prior.bootstrap_source), private=False), object_pairs_hook=unique_pairs) + if prior_config.get("original_genesis") is not None: + from .original_genesis import GENESIS_NAME, read_genesis, validate_history_genesis + + if prior_config["original_genesis"] != "/bootstrap/" + GENESIS_NAME: + raise ValueError("original genesis predecessor selection differs") + genesis = read_genesis(Path(prior.bootstrap_source).parent / GENESIS_NAME) + if validate_history_genesis(Path(genesis.binding.root)) != genesis or (genesis.instance_id, genesis.management_id, genesis.file("bootstrap").sha256) != (prior.instance_id, prior.container_id, prior.bootstrap_sha256): + raise ValueError("original genesis predecessor authority differs") + if (intent.original_history, intent.original_history_id) != (genesis.binding.root, genesis.binding.history): + raise ValueError("covered original genesis history cannot be omitted or replaced") if prior_config.get("fresh_prior_lock_path") is not None: # An explicitly enrolled original predecessor cannot silently lose its # source-bound history when the next real host cutover is prepared. diff --git a/src/auto_coder/management/fresh_host_observation.py b/src/auto_coder/management/fresh_host_observation.py index 264b36788..dc0549cc6 100644 --- a/src/auto_coder/management/fresh_host_observation.py +++ b/src/auto_coder/management/fresh_host_observation.py @@ -169,6 +169,21 @@ def capture_prior_container(container_id: str, image_id: str, *, expected_mounts raise HostObservationUnavailable("unsupported prior container containment") initialized_fresh = False allowed_destinations = {"/bootstrap", "/management-child-state", "/management-oauth", "/management-operations", "/management-policy", "/management-versions", "/run/firebase-admin", "/run/secrets/release_worker_bridge", "/run/secrets/beta_worker_bridge", "/etc/caddy/Caddyfile"} + if not require_owner_lock and not require_deployment_identity and values[15:17] == [["caddy"], ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]]: + # The stock Caddy image declares these data volumes. Its explicit + # bind-only profile must select the actual private roots, not silently + # create Docker volumes or admit executable/configuration overlays. + if isinstance(mounts, list): + data_roots = [item for item in mounts if isinstance(item, dict) and item.get("Destination") in ("/data", "/config")] + if data_roots: + if len(data_roots) != 2 or {item.get("Destination") for item in data_roots} != {"/data", "/config"}: + raise HostObservationUnavailable("Caddy data root bindings are incomplete") + for item in data_roots: + path = Path(item.get("Source", "")) + info = path.lstat() + if item.get("Type") != "bind" or item.get("RW") is not True or path != path.resolve(strict=True) or not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise HostObservationUnavailable("Caddy data root binding differs") + allowed_destinations.update(("/data", "/config")) if require_deployment_identity: from .exclusion_host import inherited_support @@ -663,6 +678,10 @@ def assert_host_listener_absent(port: int) -> None: def assert_replacement_published_listener(intent: FreshInitializationIntent) -> None: + assert_container_published_listener(intent.new_container_id, intent.image_digest, intent.host_listener_port) + + +def assert_container_published_listener(container_id: str, image_id: str, published_port: int) -> None: """Prove the host listener is Docker's proxy for this exact replacement. This intentionally supports only the Docker userland-proxy profile. Kernel @@ -672,8 +691,10 @@ def assert_replacement_published_listener(intent: FreshInitializationIntent) -> try: if os.readlink("/proc/self/ns/net") != os.readlink("/proc/1/ns/net") or os.readlink("/proc/self/ns/pid") != os.readlink("/proc/1/ns/pid"): raise ValueError("observer is not in the host namespaces") - values = _inspect(intent.new_container_id) - if values[1] != intent.image_digest or values[2] is not True or values[17] != {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(intent.host_listener_port)}]}: + if not re.fullmatch(r"[0-9a-f]{64}", container_id) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) or type(published_port) is not int or not 0 < published_port <= 65535: + raise ValueError("published endpoint selection differs") + values = _inspect(container_id) + if values[1] != image_id or values[2] is not True or values[17] != {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(published_port)}]}: raise ValueError("replacement published endpoint differs") networks = values[18] if not isinstance(networks, dict): @@ -693,7 +714,7 @@ def listener_inodes() -> set[str]: if len(fields) < 10: raise ValueError("listener table incomplete") address, port = fields[1].rsplit(":", 1) - if int(port, 16) == intent.host_listener_port and fields[3] == "0A": + if int(port, 16) == published_port and fields[3] == "0A": if table != "tcp" or address != "0100007F" or not fields[9].isdecimal(): raise ValueError("unexpected host listener") found.add(fields[9]) @@ -705,8 +726,8 @@ def listener_inodes() -> set[str]: daemon = int(Path("/var/run/docker.pid").read_text(encoding="ascii").strip()) if daemon <= 0 or os.stat(f"/proc/{daemon}/exe") != os.stat("/usr/bin/dockerd"): raise ValueError("Docker daemon identity differs") - expected_prefix = ["/usr/bin/docker-proxy", "-proto", "tcp", "-host-ip", "127.0.0.1", "-host-port", str(intent.host_listener_port), "-container-ip"] - socket_owners = subprocess.run(["ss", "-ltnpH", f"sport = :{intent.host_listener_port}"], check=True, capture_output=True, text=True, timeout=10).stdout + expected_prefix = ["/usr/bin/docker-proxy", "-proto", "tcp", "-host-ip", "127.0.0.1", "-host-port", str(published_port), "-container-ip"] + socket_owners = subprocess.run(["ss", "-ltnpH", f"sport = :{published_port}"], check=True, capture_output=True, text=True, timeout=10).stdout candidates = [int(value) for value in re.findall(r'"docker-proxy",pid=(\d+),fd=\d+', socket_owners)] if len(candidates) != 1: raise ValueError("Docker proxy is missing or ambiguous") @@ -729,7 +750,7 @@ def listener_inodes() -> set[str]: raise ValueError("replacement proxy lifetime changed") if listener_inodes() != inodes: raise ValueError("replacement listener inode changed") - final_values = _inspect(intent.new_container_id) + final_values = _inspect(container_id) # Docker reports mounts as an unordered collection. Compare every # complete mount object, including duplicate entries, independent diff --git a/src/auto_coder/management/original_attempt_evidence.py b/src/auto_coder/management/original_attempt_evidence.py index 2ae45d4b7..9ff7d63f2 100644 --- a/src/auto_coder/management/original_attempt_evidence.py +++ b/src/auto_coder/management/original_attempt_evidence.py @@ -20,6 +20,7 @@ from .fresh_initialization import FreshInitializationIntent, parse_fresh_intent, read_fresh_status from .original_attempts import PROFILE, HistoryBinding, HistoryEvent, OriginalEvidenceUnavailable, canonical, history_lock, identity, read_frontier, read_original, source_identity, validate_original_lifetime +from .original_genesis import HISTORY_PROFILE, GenesisAnchor @dataclass(frozen=True) @@ -51,6 +52,46 @@ class OriginalAttemptEvidence: # There is deliberately no completeness boolean or authorization field. +@dataclass(frozen=True) +class OriginalGeneration: + """Original scoped identities; an unresolved outcome remains unresolved.""" + + request_id: str + instance_id: str + management_id: str + image_id: str + bootstrap_config_sha256: str + management_config_sha256: str + state_root: str + state_device: int + state_inode: int + journal_root: str + journal_device: int + journal_inode: int + oauth_root: str + oauth_device: int + oauth_inode: int + owner_path: str + owner_device: int + owner_inode: int + outcome: str + expected_stores: tuple[tuple[str, str], ...] + proxy_id: str = "" + proxy_image_id: str = "" + + +@dataclass(frozen=True) +class OriginalLineageEvidence: + binding: HistoryBinding + profile: str = HISTORY_PROFILE + genesis: GenesisAnchor | None = None + frontier: str = "" + attempts: tuple[AttemptEvidence, ...] = () + generations: tuple[OriginalGeneration, ...] = () + originals: tuple[OriginalFile, ...] = () + unavailable: tuple[str, ...] = () + + def _snapshot(path: Path) -> OriginalFile: data, (device, inode) = read_original(path) return OriginalFile(str(path), device, inode, hashlib.sha256(data).hexdigest()) @@ -114,15 +155,39 @@ def _check_owner_fence(path: Path, fd: int, expected: tuple[int, int]) -> None: raise OriginalEvidenceUnavailable("consumer does not hold original-store owner fence") -def derive(binding: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalAttemptEvidence: +def _derive(binding: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = (), *, preserve_outcomes: bool = False) -> tuple[OriginalAttemptEvidence, tuple[OriginalGeneration, ...]]: """Enumerate original authority, independently of any submitted result.""" files: dict[str, OriginalFile] = {} attempts: list[AttemptEvidence] = [] + generations: list[OriginalGeneration] = [] unavailable: list[str] = [] held_owners: list[tuple[Path, int, tuple[int, int]]] = [] scanned_namespaces: dict[Path, tuple[tuple[int, int], tuple[str, ...]]] = {} frontier = "" + def generation(intent: FreshInitializationIntent, outcome: str, stores: tuple[tuple[str, str], ...], owner: tuple[int, int] = (0, 0)) -> OriginalGeneration: + return OriginalGeneration( + intent.request_id, + intent.new_instance, + intent.new_container_id, + intent.image_digest, + intent.bootstrap_config_sha256, + intent.management_config_sha256, + intent.host_state_root, + intent.state_device, + intent.state_inode, + intent.host_journal_root, + intent.journal_device, + intent.journal_inode, + intent.host_oauth_root, + intent.oauth_device, + intent.oauth_inode, + str(Path(intent.host_state_root) / ".supervisor-owner.lock"), + *owner, + outcome, + stores, + ) + def retain(path: Path) -> bytes: # Hash the exact bytes consumed by this derivation. A second read # could parse changed bytes under the first read's digest. @@ -230,6 +295,7 @@ def retain(path: Path) -> bytes: if kinds[-1] == "never-created": if anchor["pristine"] is not True or json.loads(branch[-1].value_json) != {"lifetime": anchor["lifetime"], "stores": expected} or scanned_namespaces[state][1] or scanned_namespaces[journal][1]: raise OriginalEvidenceUnavailable("never-created original boundary conflicts with effects") + generations.append(generation(intent, "never-created", tuple(sorted(expected.items())))) continue if "claimed" not in kinds: unavailable.append(attempt + ": original admission unresolved") @@ -287,11 +353,18 @@ def retain(path: Path) -> bytes: raise OriginalEvidenceUnavailable("unsupported nested evidence profile") retain(path) status = read_fresh_status(state, host=True) - if status["status"] != "succeeded": + if preserve_outcomes: + # The actual native receipt follows this original claim. + # A missing claim is a provenance gap, not an unknown outcome. + retain(state / "fresh-launch-claimed.json") + if status["status"] not in ("succeeded", "unknown", "launched"): + raise OriginalEvidenceUnavailable("original first-launch boundary unavailable") + generations.append(generation(intent, str(status["status"]), tuple(sorted(expected.items())), owner_identity)) + elif status["status"] != "succeeded": unavailable.append(attempt + ": original outcome remains " + str(status["status"])) except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as error: unavailable.append(attempt + ": " + (str(error) or type(error).__name__)) - if not events: + if not events and not (preserve_outcomes and authority.genesis_path): unavailable.append("prospective enrollment has no original attempts; legacy coverage unavailable") # Re-read every byte and identity; no consumer verdict supplies stability. for original_path, snapshot in files.items(): @@ -309,7 +382,139 @@ def retain(path: Path) -> bytes: raise OriginalEvidenceUnavailable("original source changed during derivation") except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as error: unavailable.append(str(error) or type(error).__name__) - return OriginalAttemptEvidence(binding=binding, frontier=frontier, attempts=tuple(attempts), originals=tuple(sorted(files.values(), key=lambda item: item.path)), unavailable=tuple(unavailable)) + return OriginalAttemptEvidence(binding=binding, frontier=frontier, attempts=tuple(attempts), originals=tuple(sorted(files.values(), key=lambda item: item.path)), unavailable=tuple(unavailable)), tuple(generations) + + +def derive(binding: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalAttemptEvidence: + """Keep the prospective-attempt reader's existing outcome requirements.""" + return _derive(binding, fence, owner_fences)[0] + + +def derive_lineage(binding: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalLineageEvidence: + """Join original genesis and every retained attempted successor read-only.""" + from .original_genesis import LAUNCH_NAME, NATIVE_NAME, native_paths, validate_history_genesis, validate_native_genesis + + evidence, successors = _derive(binding, fence, owner_fences, preserve_outcomes=True) + originals = {item.path: item for item in evidence.originals} + unavailable = list(evidence.unavailable) + generations: list[OriginalGeneration] = [] + genesis = None + try: + _check_fence(binding, fence) + authority, events = read_frontier(Path(binding.root), binding) + if not authority.genesis_path or authority.profile != HISTORY_PROFILE: + raise OriginalEvidenceUnavailable("full retained lineage requires original prospective genesis; earlier history remains unavailable") + path = Path(authority.genesis_path) + genesis = validate_history_genesis(Path(binding.root)) + if genesis.binding != binding or binding.source != source_identity(): + raise OriginalEvidenceUnavailable("original genesis source/history differs") + state, journal, oauth = (genesis.root(role) for role in ("state", "journal", "oauth")) + owner_path = Path(state.host_path) / ".supervisor-owner.lock" + owner_identity = identity(owner_path) + matching = [fd for fd in owner_fences if (os.fstat(fd).st_dev, os.fstat(fd).st_ino) == owner_identity] + if len(matching) != 1: + raise OriginalEvidenceUnavailable("current original genesis owner fence unavailable") + _check_owner_fence(owner_path, matching[0], owner_identity) + snapshots = [path, owner_path, Path(state.host_path) / NATIVE_NAME, Path(state.host_path) / LAUNCH_NAME] + snapshots.extend(Path(item.path) for item in genesis.files) + namespaces = {} + for directory in (Path(state.host_path), Path(journal.host_path)): + namespaces[directory] = (identity(directory, directory=True), tuple(sorted(item.name for item in directory.iterdir()))) + for item in sorted(directory.iterdir()): + if directory == Path(state.host_path) and item.name == "logs": + info = item.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid(): + raise OriginalEvidenceUnavailable("original genesis diagnostics directory differs") + continue + if item.is_symlink() or item.is_dir(): + raise OriginalEvidenceUnavailable("unsupported nested original genesis evidence") + snapshots.append(item) + for original_path in snapshots: + observed = _snapshot(original_path) + if observed.path in originals and originals[observed.path] != observed: + raise OriginalEvidenceUnavailable("original genesis changed during derivation") + if any(item.path != observed.path and (item.device, item.inode) == (observed.device, observed.inode) for item in originals.values()): + raise OriginalEvidenceUnavailable("original lineage file identity aliases") + originals[observed.path] = observed + validate_native_genesis(genesis, path, host=True) + stores = native_paths(genesis, host=True) + for store in stores.values(): + _database(store) + # This is a factual original launch boundary. It never upgrades child, + # handoff, recovery or management-operation unknowns into success. + with closing(sqlite3.connect(stores["child"].as_uri() + "?mode=ro&immutable=1", uri=True)) as connection: + rows = connection.execute("SELECT instance_id,record_json FROM child_state").fetchall() + if len(rows) > 1 or rows and rows[0][0] != genesis.instance_id: + raise OriginalEvidenceUnavailable("original genesis child identity differs") + outcome = "unknown" if not rows else json.loads(rows[0][1])["phase"] + if outcome not in {"stopped", "starting", "running", "draining", "unknown", "needs_attention"}: + raise OriginalEvidenceUnavailable("original genesis child outcome is malformed") + generations.append( + OriginalGeneration( + genesis.authorization_id, + genesis.instance_id, + genesis.management_id, + genesis.management_image_id, + genesis.file("bootstrap").sha256, + genesis.file("management").sha256, + state.host_path, + state.device, + state.inode, + journal.host_path, + journal.device, + journal.inode, + oauth.host_path, + oauth.device, + oauth.inode, + str(owner_path), + *owner_identity, + outcome, + tuple(sorted((name, str(store)) for name, store in stores.items())), + genesis.proxy_id, + genesis.proxy_image_id, + ) + ) + complete = (generations[0], *successors) + expected_owners = {(item.owner_device, item.owner_inode) for item in complete if item.owner_inode > 0} + observed_owners = [(os.fstat(fd).st_dev, os.fstat(fd).st_ino) for fd in owner_fences] + if len(observed_owners) != len(expected_owners) or set(observed_owners) != expected_owners: + raise OriginalEvidenceUnavailable("original lineage owner descriptor set differs") + by_instance = {item.instance_id: item for item in complete} + for event in events: + if event.kind != "anchor": + continue + intent = parse_fresh_intent(json.loads(event.value_json)["intent"]) + generation = by_instance.get(intent.new_instance) + if generation is None or generation.outcome == "never-created": + continue + predecessor = by_instance.get(intent.old_instance) + if predecessor is None or (intent.old_container_id, intent.owner_lock_device, intent.owner_lock_inode) != (predecessor.management_id, predecessor.owner_device, predecessor.owner_inode): + raise OriginalEvidenceUnavailable("original native successor owner differs from its retained predecessor") + if (intent.host_oauth_root, intent.oauth_device, intent.oauth_inode) != (oauth.host_path, oauth.device, oauth.inode) or identity(Path(intent.host_oauth_root), directory=True) != (oauth.device, oauth.inode): + raise OriginalEvidenceUnavailable("original native successor retained OAuth root differs") + for original_name, expected in originals.items(): + if _snapshot(Path(original_name)) != expected: + raise OriginalEvidenceUnavailable("original lineage changed during derivation") + for directory, (expected_id, names) in namespaces.items(): + if identity(directory, directory=True) != expected_id or tuple(sorted(item.name for item in directory.iterdir())) != names: + raise OriginalEvidenceUnavailable("original genesis namespace changed during derivation") + if read_frontier(Path(binding.root), binding) != (authority, events) or validate_history_genesis(Path(binding.root)) != genesis: + raise OriginalEvidenceUnavailable("original lineage advanced during derivation") + _check_fence(binding, fence) + _check_owner_fence(owner_path, matching[0], owner_identity) + except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as error: + unavailable.append(str(error) or type(error).__name__) + return OriginalLineageEvidence(binding, HISTORY_PROFILE, genesis, evidence.frontier, evidence.attempts, tuple(generations) + successors, tuple(sorted(originals.values(), key=lambda item: item.path)), tuple(unavailable)) + + +def verify_lineage(submitted: OriginalLineageEvidence, expected: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalLineageEvidence: + """Independently reconstruct all originals at the consumer's current fence.""" + current = derive_lineage(expected, fence, owner_fences) + if current != submitted or submitted.binding != expected: + raise OriginalEvidenceUnavailable("submitted lineage is forged, stale, truncated or cross-deployment") + if current.unavailable: + raise OriginalEvidenceUnavailable("; ".join(current.unavailable)) + return current def verify(submitted: OriginalAttemptEvidence, expected: HistoryBinding, fence: int, owner_fences: tuple[int, ...] = ()) -> OriginalAttemptEvidence: diff --git a/src/auto_coder/management/original_attempts.py b/src/auto_coder/management/original_attempts.py index d0cdf3e06..3c8c08e37 100644 --- a/src/auto_coder/management/original_attempts.py +++ b/src/auto_coder/management/original_attempts.py @@ -44,6 +44,7 @@ class HistoryAuthority: profile: str root: tuple[int, int] lock: tuple[int, int] + genesis_path: str = "" @dataclass(frozen=True) @@ -187,7 +188,15 @@ def read_frontier(root: Path, expected: HistoryBinding | None = None) -> tuple[H """Validate the whole bounded append-only event namespace, never truncate.""" raw, _ = read_original(root / "authority.json") authority = json.loads(raw) - if not isinstance(authority, dict) or set(authority) != {"binding", "profile", "root", "lock"} or authority["profile"] != PROFILE or canonical(authority) != raw: + from .original_genesis import HISTORY_PROFILE, validate_history_genesis + + genesis = None + if isinstance(authority, dict) and authority.get("profile") == HISTORY_PROFILE: + genesis = validate_history_genesis(root) + fields = {"binding", "profile", "root", "lock", "genesis"} + else: + fields = {"binding", "profile", "root", "lock"} + if not isinstance(authority, dict) or set(authority) != fields or authority["profile"] not in (PROFILE, HISTORY_PROFILE) or canonical(authority) != raw: raise OriginalEvidenceUnavailable("unsupported history profile") binding = HistoryBinding(**authority["binding"]) if binding.root != str(root) or (expected is not None and binding != expected) or authority["root"] != list(identity(root, directory=True)) or authority["lock"] != list(identity(root / "history.lock")): @@ -289,10 +298,10 @@ def read_frontier(root: Path, expected: HistoryBinding | None = None) -> tuple[H # All prior anchors must form one admissible forward history, including # failed branches. Checking only a new claim against its immediate parent # would accept an unrelated rewritten ancestor elsewhere in the frontier. - baseline: tuple[str, str] | None = None - known_predecessors: set[tuple[str, str]] = set() - seen_targets: set[str] = set() - seen_roots: set[tuple[int, int]] = set() + baseline: tuple[str, str] | None = (genesis.instance_id, genesis.management_id) if genesis is not None else None + known_predecessors: set[tuple[str, str]] = {baseline} if baseline is not None else set() + seen_targets: set[str] = {item.host_path for item in genesis.roots} if genesis is not None else set() + seen_roots: set[tuple[int, int]] = {(item.device, item.inode) for item in genesis.roots} if genesis is not None else set() earlier_predecessor_names: set[str] = set() for event in events: if event.kind == "claimed": @@ -330,7 +339,7 @@ def read_frontier(root: Path, expected: HistoryBinding | None = None) -> tuple[H if previous != frontier["digest"]: raise OriginalEvidenceUnavailable("truncated original frontier") - return HistoryAuthority(binding, authority["profile"], tuple(authority["root"]), tuple(authority["lock"])), events + return HistoryAuthority(binding, authority["profile"], tuple(authority["root"]), tuple(authority["lock"]), authority["genesis"]["path"] if genesis is not None else ""), events def _append(root: Path, kind: str, attempt: str, value: object) -> None: @@ -377,13 +386,18 @@ def capture_claim(intent: FreshInitializationIntent) -> Iterator[None]: if authority.binding.history != intent.original_history_id or authority.binding.source != source_identity(): raise OriginalEvidenceUnavailable("original history source or enrollment differs") anchors = [event for event in events if event.kind == "anchor"] + genesis = None + if authority.genesis_path: + from .original_genesis import read_genesis + + genesis = read_genesis(Path(authority.genesis_path)) keys = {intent.new_instance, intent.new_container_id, intent.host_state_root, intent.host_journal_root} prior_targets = [json.loads(event.value_json)["intent"] for event in anchors] if any(event.attempt == intent.request_id for event in anchors) or any(keys & {prior["new_instance"], prior["new_container_id"], prior["host_state_root"], prior["host_journal_root"]} for prior in prior_targets): raise OriginalEvidenceUnavailable("original attempt or target already anchored; no replay") - if anchors: - baseline = json.loads(anchors[0].value_json)["intent"] - predecessors = [(baseline["old_instance"], baseline["old_container_id"])] + if anchors or genesis is not None: + baseline_pair = (genesis.instance_id, genesis.management_id) if genesis is not None else (json.loads(anchors[0].value_json)["intent"]["old_instance"], json.loads(anchors[0].value_json)["intent"]["old_container_id"]) + predecessors = [baseline_pair] for prior_event in anchors: prior_kinds = [event.kind for event in events if event.attempt == prior_event.attempt] # A refused, interrupted or merely anchored attempt never established @@ -391,7 +405,7 @@ def capture_claim(intent: FreshInitializationIntent) -> Iterator[None]: if prior_kinds in (["anchor", "claimed"], ["anchor", "claimed", "grant-possible"]): prior = json.loads(prior_event.value_json)["intent"] predecessors.append((prior["new_instance"], prior["new_container_id"])) - if (intent.old_instance, intent.old_container_id) not in predecessors or intent.new_instance == baseline["old_instance"] or intent.new_container_id == baseline["old_container_id"]: + if (intent.old_instance, intent.old_container_id) not in predecessors or intent.new_instance == baseline_pair[0] or intent.new_container_id == baseline_pair[1]: raise OriginalEvidenceUnavailable("uncovered, refused or cyclic original predecessor") if any(prior["old_instance"] == intent.new_instance for prior in prior_targets): raise OriginalEvidenceUnavailable("attempt would cycle covered ancestry") @@ -402,6 +416,8 @@ def capture_claim(intent: FreshInitializationIntent) -> Iterator[None]: if state_id != (intent.state_device, intent.state_inode) or journal_id != (intent.journal_device, intent.journal_inode): raise OriginalEvidenceUnavailable("intended root identities differ before anchor") prior_root_ids = {tuple(json.loads(event.value_json)[field]) for event in anchors for field in ("state", "journal")} + if genesis is not None: + prior_root_ids.update((item.device, item.inode) for item in genesis.roots) if state_id == journal_id or state_id in prior_root_ids or journal_id in prior_root_ids: raise OriginalEvidenceUnavailable("original target aliases a protected mounted root") pristine = not any(Path(intent.host_state_root).iterdir()) and not any(Path(intent.host_journal_root).iterdir()) diff --git a/src/auto_coder/management/original_genesis.py b/src/auto_coder/management/original_genesis.py new file mode 100644 index 000000000..00573c371 --- /dev/null +++ b/src/auto_coder/management/original_genesis.py @@ -0,0 +1,592 @@ +"""Prospective ordinary-deployment originals, never recovery or launch authority. + +A separately authorized capture observes two never-started local Docker objects. +The existing ordinary Supervisor records native store and first-launch boundaries +under its own original exclusive lock. Neither entry point converts old roots. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sqlite3 +import stat +import sys +import uuid +from contextlib import closing +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from loguru import logger + +from .original_attempts import HistoryBinding, OriginalEvidenceUnavailable, _create, _lifetime, _sync, canonical, history_lock, identity, read_original, source_identity, validate_original_lifetime + +if TYPE_CHECKING: + from .bootstrap_supervisor import BootstrapConfig, ManagementBootstrapSupervisor + +PROFILE = "ordinary-original-genesis-v1" +HISTORY_PROFILE = "ordinary-original-history-v1" +GENESIS_NAME = "original-genesis.json" +NATIVE_NAME = "original-genesis-stores.json" +LAUNCH_NAME = "original-genesis-first-launch.json" +HISTORY_MOUNT = Path("/management-original-history") +ROOTS_PROFILE = "ordinary-original-roots-v1" +ROOTS_NAME = "original-genesis-roots.json" +ROOT_CLAIM_NAME = "original-genesis-root-claim.json" + + +@dataclass(frozen=True) +class GenesisRootAuthorization: + """Select prospective paths before the first Docker object exists.""" + + profile: str + purpose: str + deployment_id: str + authorization_id: str + bootstrap_path: str + management_path: str + proxy_config_path: str + state_root: str + journal_root: str + oauth_root: str + history_root: str + + +@dataclass(frozen=True) +class GenesisSelection: + """The protected separate new-deployment authorization, not a verdict.""" + + profile: str + purpose: str + deployment_id: str + authorization_id: str + management_id: str + proxy_id: str + history_root: str + bootstrap_path: str + management_path: str + proxy_config_path: str + root_authority_path: str + + +@dataclass(frozen=True) +class GenesisFile: + role: str + path: str + device: int + inode: int + sha256: str + + +@dataclass(frozen=True) +class GenesisRoot: + role: str + host_path: str + container_path: str + device: int + inode: int + + +@dataclass(frozen=True) +class GenesisAnchor: + profile: str + binding: HistoryBinding + authorization_id: str + instance_id: str + engine_id: str + management_id: str + management_image_id: str + management_inspection_sha256: str + proxy_id: str + proxy_image_id: str + proxy_inspection_sha256: str + files: tuple[GenesisFile, ...] + roots: tuple[GenesisRoot, ...] + lifetime_json: str + + def file(self, role: str) -> GenesisFile: + matches = [item for item in self.files if item.role == role] + if len(matches) != 1: + raise OriginalEvidenceUnavailable("original genesis file role is incomplete") + return matches[0] + + def root(self, role: str) -> GenesisRoot: + matches = [item for item in self.roots if item.role == role] + if len(matches) != 1: + raise OriginalEvidenceUnavailable("original genesis root role is incomplete") + return matches[0] + + +def _file(role: str, path: Path) -> GenesisFile: + encoded, (device, inode) = read_original(path) + return GenesisFile(role, str(path), device, inode, hashlib.sha256(encoded).hexdigest()) + + +def _require_host_root() -> None: + if os.geteuid() != 0: + raise OriginalEvidenceUnavailable("ordinary original genesis producer requires host root") + + +def prepare_original_genesis_roots(authorization_path: Path) -> Path: + """Create the authorized original roots once, without any Docker effects. + + Preexisting paths are never enrolled. A partial claim remains permanently + unavailable, including when the directories it created still look empty. + """ + _require_host_root() + encoded, _ = read_original(authorization_path) + value = json.loads(encoded) + if not isinstance(value, dict) or set(value) != set(GenesisRootAuthorization.__dataclass_fields__) or canonical(value) != encoded or any(not isinstance(item, str) for item in value.values()): + raise OriginalEvidenceUnavailable("original root authorization fields differ") + request = GenesisRootAuthorization(**value) + if request.profile != ROOTS_PROFILE or request.purpose != "separately-authorized-new-deployment" or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", request.deployment_id) or str(uuid.UUID(request.authorization_id)) != request.authorization_id: + raise OriginalEvidenceUnavailable("explicit new-root authorization required") + bootstrap = Path(request.bootstrap_path).parent + identity(bootstrap, directory=True) + configurations = tuple(_file(role, Path(selected)) for role, selected in (("bootstrap", request.bootstrap_path), ("management", request.management_path), ("proxy", request.proxy_config_path))) + paths = tuple((role, Path(getattr(request, role + "_root")), container) for role, container in (("state", "/management-child-state"), ("journal", "/management-operations"), ("oauth", "/management-oauth"), ("history", str(HISTORY_MOUNT)))) + if any(a == b or a in b.parents or b in a.parents for index, a in enumerate([bootstrap, *(path for _, path, _ in paths)]) for b in [bootstrap, *(path for _, path, _ in paths)][index + 1 :]): + raise OriginalEvidenceUnavailable("authorized original root paths overlap") + for _, path, _ in paths: + if not path.is_absolute() or path != path.resolve() or path.exists() or path.is_symlink(): + raise OriginalEvidenceUnavailable("original root already exists; no empty-root conversion") + identity(path.parent, directory=True) + original_authorization = _file("root-authorization", authorization_path) + claim = {"profile": ROOTS_PROFILE, "authorization": asdict(original_authorization), "source": source_identity(), "request": value, "lifetime": _lifetime(), "configurations": [asdict(item) for item in configurations]} + _create(bootstrap, ROOT_CLAIM_NAME, claim) + roots = [] + for role, path, container in paths: + path.mkdir(mode=0o700) + _sync(path) + _sync(path.parent) + roots.append(GenesisRoot(role, str(path), container, *identity(path, directory=True))) + if tuple(_file(item.role, Path(item.path)) for item in configurations) != configurations or _file("root-authorization", authorization_path) != original_authorization: + raise OriginalEvidenceUnavailable("original root authorization changed during creation") + _create(bootstrap, ROOTS_NAME, {"profile": ROOTS_PROFILE, "claim": asdict(_file("root-claim", bootstrap / ROOT_CLAIM_NAME)), "roots": [asdict(item) for item in roots]}) + _read_root_authority(bootstrap / ROOTS_NAME) + logger.bind(original_boundary="genesis-roots", evidence_only=True).info("Original separately authorized deployment roots created: deployment={}", request.deployment_id) + return bootstrap / ROOTS_NAME + + +def _read_root_authority(path: Path) -> tuple[GenesisRootAuthorization, tuple[GenesisRoot, ...], tuple[GenesisFile, ...]]: + encoded, _ = read_original(path) + value = json.loads(encoded) + if not isinstance(value, dict) or set(value) != {"profile", "claim", "roots"} or value["profile"] != ROOTS_PROFILE or canonical(value) != encoded: + raise OriginalEvidenceUnavailable("original root creation receipt incomplete") + claim_file = GenesisFile(**value["claim"]) + if claim_file.role != "root-claim" or Path(claim_file.path) != path.parent / ROOT_CLAIM_NAME or _file("root-claim", Path(claim_file.path)) != claim_file: + raise OriginalEvidenceUnavailable("original root creation claim differs") + claim_encoded, _ = read_original(Path(claim_file.path)) + claim = json.loads(claim_encoded) + if not isinstance(claim, dict) or set(claim) != {"profile", "authorization", "source", "request", "lifetime", "configurations"} or claim["profile"] != ROOTS_PROFILE or canonical(claim) != claim_encoded or claim["source"] != source_identity(): + raise OriginalEvidenceUnavailable("original root producer source or fields differ") + validate_original_lifetime(claim["lifetime"]) + authorization = GenesisFile(**claim["authorization"]) + if authorization.role != "root-authorization" or _file("root-authorization", Path(authorization.path)) != authorization: + raise OriginalEvidenceUnavailable("original root authorization changed") + request = GenesisRootAuthorization(**claim["request"]) + if read_original(Path(authorization.path))[0] != canonical(asdict(request)) or path != Path(request.bootstrap_path).parent / ROOTS_NAME: + raise OriginalEvidenceUnavailable("original root authorization scope differs") + roots = tuple(GenesisRoot(**item) for item in value["roots"]) + if len(roots) != 4 or {item.role for item in roots} != {"state", "journal", "oauth", "history"} or len({(item.device, item.inode) for item in roots}) != 4: + raise OriginalEvidenceUnavailable("original root creation set is incomplete or aliased") + destinations = {"state": "/management-child-state", "journal": "/management-operations", "oauth": "/management-oauth", "history": str(HISTORY_MOUNT)} + for root in roots: + if root.host_path != getattr(request, root.role + "_root") or root.container_path != destinations[root.role] or identity(Path(root.host_path), directory=True) != (root.device, root.inode): + raise OriginalEvidenceUnavailable("original created root changed") + configurations = tuple(GenesisFile(**item) for item in claim["configurations"]) + if len(configurations) != 3 or {item.role for item in configurations} != {"bootstrap", "management", "proxy"}: + raise OriginalEvidenceUnavailable("original root configuration set is incomplete") + for item in configurations: + selected = request.proxy_config_path if item.role == "proxy" else getattr(request, item.role + "_path") + if item.path != selected or _file(item.role, Path(item.path)) != item: + raise OriginalEvidenceUnavailable("original root configuration changed") + return request, roots, (authorization, claim_file, _file("roots", path)) + + +def read_selection(path: Path) -> GenesisSelection: + encoded, _ = read_original(path) + value = json.loads(encoded) + if not isinstance(value, dict) or set(value) != set(GenesisSelection.__dataclass_fields__) or canonical(value) != encoded or any(not isinstance(item, str) for item in value.values()): + raise OriginalEvidenceUnavailable("genesis authorization fields or encoding differ") + selection = GenesisSelection(**value) + if selection.profile != PROFILE or selection.purpose != "separately-authorized-new-deployment" or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", selection.deployment_id) or str(uuid.UUID(selection.authorization_id)) != selection.authorization_id: + raise OriginalEvidenceUnavailable("explicit prospective new-deployment authorization required") + if any(not re.fullmatch(r"[0-9a-f]{64}", value) for value in (selection.management_id, selection.proxy_id)) or selection.management_id == selection.proxy_id: + raise OriginalEvidenceUnavailable("distinct exact original Docker IDs required") + for selected in (selection.history_root, selection.bootstrap_path, selection.management_path, selection.proxy_config_path, selection.root_authority_path): + candidate = Path(selected) + if not candidate.is_absolute() or candidate != candidate.resolve(strict=True): + raise OriginalEvidenceUnavailable("original genesis selection is not canonical") + return selection + + +def read_genesis(path: Path) -> GenesisAnchor: + encoded, _ = read_original(path) + raw = json.loads(encoded) + if not isinstance(raw, dict) or set(raw) != set(GenesisAnchor.__dataclass_fields__) or canonical(raw) != encoded or raw.get("profile") != PROFILE: + raise OriginalEvidenceUnavailable("original genesis schema or encoding differs") + if not isinstance(raw["binding"], dict) or set(raw["binding"]) != set(HistoryBinding.__dataclass_fields__) or any(not isinstance(value, str) for value in raw["binding"].values()): + raise OriginalEvidenceUnavailable("original genesis history binding differs") + binding = HistoryBinding(**raw["binding"]) + if str(uuid.UUID(binding.history)) != binding.history or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", binding.deployment) or not re.fullmatch(r"sha256:[0-9a-f]{64}", binding.source): + raise OriginalEvidenceUnavailable("original genesis source or history identity differs") + for key in ("management_id", "proxy_id", "management_inspection_sha256", "proxy_inspection_sha256"): + if not isinstance(raw[key], str) or not re.fullmatch(r"[0-9a-f]{64}", raw[key]): + raise OriginalEvidenceUnavailable("original genesis Docker identity differs") + for key in ("management_image_id", "proxy_image_id"): + if not isinstance(raw[key], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", raw[key]): + raise OriginalEvidenceUnavailable("original genesis image identity differs") + if raw["management_id"] == raw["proxy_id"] or any(not isinstance(raw[key], str) or not raw[key] or len(raw[key]) > 128 for key in ("instance_id", "engine_id")) or str(uuid.UUID(raw["authorization_id"])) != raw["authorization_id"]: + raise OriginalEvidenceUnavailable("original genesis deployment identity differs") + files = [] + if not isinstance(raw["files"], list) or len(raw["files"]) != 7: + raise OriginalEvidenceUnavailable("original genesis original files are incomplete") + for record in raw["files"]: + if not isinstance(record, dict) or set(record) != set(GenesisFile.__dataclass_fields__): + raise OriginalEvidenceUnavailable("original genesis original file fields differ") + if any(type(record[key]) is not int or record[key] < (1 if key == "inode" else 0) for key in ("device", "inode")) or not isinstance(record["path"], str) or not Path(record["path"]).is_absolute() or not isinstance(record["sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}", record["sha256"]): + raise OriginalEvidenceUnavailable("original genesis original file identity differs") + files.append(GenesisFile(**record)) + roots = [] + if not isinstance(raw["roots"], list) or len(raw["roots"]) != 5: + raise OriginalEvidenceUnavailable("original genesis root set is incomplete") + for record in raw["roots"]: + if not isinstance(record, dict) or set(record) != set(GenesisRoot.__dataclass_fields__): + raise OriginalEvidenceUnavailable("original genesis root fields differ") + if any(type(record[key]) is not int or record[key] < (1 if key == "inode" else 0) for key in ("device", "inode")) or any(not isinstance(record[key], str) or not Path(record[key]).is_absolute() for key in ("host_path", "container_path")): + raise OriginalEvidenceUnavailable("original genesis root identity differs") + roots.append(GenesisRoot(**record)) + if {item.role for item in files} != {"authorization", "bootstrap", "management", "proxy", "root-authorization", "root-claim", "roots"} or {item.role for item in roots} != {"bootstrap", "state", "journal", "oauth", "history"}: + raise OriginalEvidenceUnavailable("original genesis roles are incomplete or duplicated") + if len({(item.device, item.inode) for item in files}) != len(files) or len({(item.device, item.inode) for item in roots}) != len(roots): + raise OriginalEvidenceUnavailable("original genesis identities alias") + if not isinstance(raw["lifetime_json"], str): + raise OriginalEvidenceUnavailable("original genesis producer lifetime missing") + validate_original_lifetime(json.loads(raw["lifetime_json"])) + return GenesisAnchor(**{**raw, "binding": binding, "files": tuple(files), "roots": tuple(roots)}) + + +def _created_pair(selection: GenesisSelection): + """Read the full local engine and the exact finite ordinary command profile.""" + from .containment_observation import ProposedScope, _docker + from .exclusion_host import _mount + from .fresh_host_observation import _supported_prior_command + from .installed_bootstrap_mount_profile import _inspect, _mounts, _unshadowed_bootstrap_execution + from .recovery_source_profile import read_image_source_identity + + scope = ProposedScope(selection.deployment_id, selection.authorization_id, selection.authorization_id, "ordinary-v1", selection.management_id, selection.proxy_id) + observed = _docker(scope) + selected = {item.container_id: item for item in observed if item.container_id in (selection.management_id, selection.proxy_id)} + if any(item.management_route_roles for item in observed if item.container_id not in selected): + raise OriginalEvidenceUnavailable("another original management/proxy Docker route is present") + for item in selected.values(): + if (item.status, item.running, item.pid, item.started_at, item.finished_at, item.restart_count, item.restart_policy, item.maximum_retry_count) != ("created", False, 0, "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", 0, "no", 0): + raise OriginalEvidenceUnavailable("genesis requires exact created and never-started Docker objects") + management, proxy = _inspect(selection.management_id), _inspect(selection.proxy_id) + for object_id, inspected in ((selection.management_id, management), (selection.proxy_id, proxy)): + if hashlib.sha256(canonical([inspected])).hexdigest() != selected[object_id].inspection_sha256: + raise OriginalEvidenceUnavailable("Docker object changed during original capture") + host, config = inspected.get("HostConfig"), inspected.get("Config") + if not isinstance(host, dict) or not isinstance(config, dict) or host.get("Privileged") is not False or host.get("PidMode") not in ("", None) or host.get("CapAdd") not in (None, []) or host.get("Devices") not in (None, []) or config.get("User") != "": + raise OriginalEvidenceUnavailable("ordinary genesis container execution profile differs") + config = management["Config"] + assert isinstance(config, dict) + if not _supported_prior_command(config.get("Entrypoint"), config.get("Cmd")): + raise OriginalEvidenceUnavailable("genesis requires the ordinary bootstrap command") + mounts = _mounts(management) + _unshadowed_bootstrap_execution(config, mounts) + raw_mounts = management["Mounts"] + assert isinstance(raw_mounts, list) + if _mount(raw_mounts, "/bootstrap") / "config.json" != Path(selection.bootstrap_path) or _mount(raw_mounts, str(HISTORY_MOUNT)) != Path(selection.history_root): + raise OriginalEvidenceUnavailable("original genesis bootstrap or history mount differs") + if Path(selection.management_path) != Path(selection.bootstrap_path).parent / "management.yaml": + raise OriginalEvidenceUnavailable("original genesis management file is not the fixed bootstrap member") + proxy_config, proxy_host = proxy["Config"], proxy["HostConfig"] + assert isinstance(proxy_config, dict) and isinstance(proxy_host, dict) + if proxy_config.get("Entrypoint") != ["caddy"] or proxy_config.get("Cmd") != ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] or proxy_host.get("NetworkMode") != "container:" + selection.management_id: + raise OriginalEvidenceUnavailable("original genesis proxy is not the exact ordinary paired command") + proxy_mounts = proxy["Mounts"] + assert isinstance(proxy_mounts, list) + if _mount(proxy_mounts, "/etc/caddy/Caddyfile") != Path(selection.proxy_config_path): + raise OriginalEvidenceUnavailable("original genesis proxy configuration mount differs") + actual_source = read_image_source_identity(selection.management_id) + if actual_source != source_identity(): + raise OriginalEvidenceUnavailable("created original bootstrap source differs from executing producer") + return observed, management, actual_source + + +def _validate_management_settings(config: BootstrapConfig, management_path: Path, bootstrap_path: Path) -> None: + """Bind the service's effective stores to the native original receipts.""" + from .oauth_config import load_management_settings, management_listener_binding_matches + + settings = load_management_settings(management_path) + if settings.bootstrap_config_path != bootstrap_path or settings.state_root != config.oauth_root or settings.operation_state_root != config.journal_root: + raise OriginalEvidenceUnavailable("ordinary genesis effective management roots differ") + if not management_listener_binding_matches(config, settings): + raise OriginalEvidenceUnavailable("ordinary genesis effective management listener differs") + + +def capture_original_genesis(authorization_path: Path) -> GenesisAnchor: + """Capture only; this operation never starts, configures or stops Docker.""" + from .bootstrap_supervisor import BootstrapConfig + from .exclusion_host import _mount + + _require_host_root() + selection = read_selection(authorization_path) + request, created_roots, root_files = _read_root_authority(Path(selection.root_authority_path)) + if any(getattr(request, name) != getattr(selection, name) for name in ("purpose", "deployment_id", "authorization_id", "history_root", "bootstrap_path", "management_path", "proxy_config_path")): + raise OriginalEvidenceUnavailable("original root authorization differs from selected deployment") + config = BootstrapConfig.read(Path(selection.bootstrap_path)) + if config.original_genesis != Path("/bootstrap") / GENESIS_NAME or config.fresh_guard_path is not None or config.new_instance_initialization or config.archived_exclusions: + raise OriginalEvidenceUnavailable("ordinary bootstrap did not explicitly select original genesis") + if (str(config.state_root), str(config.journal_root), str(config.oauth_root)) != ("/management-child-state", "/management-operations", "/management-oauth"): + raise OriginalEvidenceUnavailable("ordinary genesis retained root mapping differs") + runtime = next(item for item in config.runtimes if item.reference == config.initial_runtime) + if runtime.executable != Path("/opt/auto-coder/.venv/bin/auto-coder") or runtime.arguments != ("management", "serve", "--config", "/bootstrap/management.yaml") or runtime.execution_kind != "source-script" or config.management_config not in (None, Path("/bootstrap/management.yaml")): + raise OriginalEvidenceUnavailable("ordinary genesis management runtime differs") + _validate_management_settings(config, Path(selection.management_path), Path("/bootstrap/config.json")) + files = (*tuple(_file(role, path) for role, path in (("authorization", authorization_path), ("bootstrap", Path(selection.bootstrap_path)), ("management", Path(selection.management_path)), ("proxy", Path(selection.proxy_config_path)))), *root_files) + observed, management, source = _created_pair(selection) + mounts = management["Mounts"] + assert isinstance(mounts, list) + roots = tuple( + GenesisRoot(role, str(host), str(container), *identity(host, directory=True)) + for role, host, container in ( + ("bootstrap", Path(selection.bootstrap_path).parent, Path("/bootstrap")), + ("state", _mount(mounts, str(config.state_root), rw=True), config.state_root), + ("journal", _mount(mounts, str(config.journal_root), rw=True), config.journal_root), + ("oauth", _mount(mounts, str(config.oauth_root), rw=True), config.oauth_root), + ("history", Path(selection.history_root), HISTORY_MOUNT), + ) + ) + host_roots = [Path(item.host_path) for item in roots] + if tuple(item for item in roots if item.role != "bootstrap") != created_roots: + raise OriginalEvidenceUnavailable("Docker mounts differ from affirmative original root creation") + if any(a == b or a in b.parents or b in a.parents for index, a in enumerate(host_roots) for b in host_roots[index + 1 :]): + raise OriginalEvidenceUnavailable("original genesis roots overlap") + for root in roots: + if root.role != "bootstrap" and any(Path(root.host_path).iterdir()): + raise OriginalEvidenceUnavailable("new separately authorized genesis roots must be empty") + genesis_path = Path(selection.bootstrap_path).parent / GENESIS_NAME + if genesis_path.exists() or genesis_path.is_symlink(): + raise OriginalEvidenceUnavailable("original genesis already captured or interrupted; no reenrollment") + history_root = Path(selection.history_root) + # The first durable claim serializes capture and permanently poisons a partial + # publication. Existing roots never acquire a replacement original authority. + _create(history_root, "history.lock", {}) + with history_lock(history_root, exclusive=True): + binding = HistoryBinding(str(history_root), selection.deployment_id, str(uuid.uuid4()), source) + selected = {item.container_id: item for item in observed} + manager, proxy = selected[selection.management_id], selected[selection.proxy_id] + anchor = GenesisAnchor(PROFILE, binding, selection.authorization_id, config.instance_id, manager.engine_id, manager.container_id, manager.image_id, manager.inspection_sha256, proxy.container_id, proxy.image_id, proxy.inspection_sha256, files, roots, canonical(_lifetime()).decode()) + if _created_pair(selection) != (observed, management, source) or tuple(_file(item.role, Path(item.path)) for item in files) != files: + raise OriginalEvidenceUnavailable("original genesis inputs changed during capture") + for root in roots: + if identity(Path(root.host_path), directory=True) != (root.device, root.inode): + raise OriginalEvidenceUnavailable("original genesis root changed during capture") + allowed = {"history.lock"} if root.role == "history" else set() + if root.role != "bootstrap" and {item.name for item in Path(root.host_path).iterdir()} != allowed: + raise OriginalEvidenceUnavailable("original genesis root acquired earlier records") + _create(genesis_path.parent, genesis_path.name, asdict(anchor)) + original = _file("genesis", genesis_path) + _create(history_root, "authority.json", {"binding": asdict(binding), "profile": HISTORY_PROFILE, "root": identity(history_root, directory=True), "lock": identity(history_root / "history.lock"), "genesis": asdict(original)}) + _create(history_root, "frontier.json", {"count": 0, "digest": ""}) + validate_history_genesis(history_root) + logger.bind(original_history=str(history_root), original_boundary="genesis", evidence_only=True).info("Original ordinary deployment genesis captured: deployment={} history={}", binding.deployment, binding.history) + return anchor + + +def validate_history_genesis(root: Path, *, mounted_genesis: Path | None = None) -> GenesisAnchor: + """Read the original authority and its exact genesis, with no store creation.""" + encoded, _ = read_original(root / "authority.json") + value = json.loads(encoded) + if not isinstance(value, dict) or set(value) != {"binding", "profile", "root", "lock", "genesis"} or value.get("profile") != HISTORY_PROFILE or canonical(value) != encoded: + raise OriginalEvidenceUnavailable("ordinary original history authority is incomplete") + original = value["genesis"] + if not isinstance(original, dict) or set(original) != set(GenesisFile.__dataclass_fields__) or original.get("role") != "genesis": + raise OriginalEvidenceUnavailable("ordinary original genesis pointer is incomplete") + path = mounted_genesis or Path(original["path"]) + actual = _file("genesis", path) + if (actual.device, actual.inode, actual.sha256) != (original["device"], original["inode"], original["sha256"]) or mounted_genesis is None and actual.path != original["path"]: + raise OriginalEvidenceUnavailable("original genesis inode or bytes changed") + anchor = read_genesis(path) + if value["binding"] != asdict(anchor.binding) or value["root"] != list(identity(root, directory=True)) or value["lock"] != list(identity(root / "history.lock")): + raise OriginalEvidenceUnavailable("original genesis/history identities differ") + history = anchor.root("history") + if (history.device, history.inode) != identity(root, directory=True) or history.host_path != anchor.binding.root: + raise OriginalEvidenceUnavailable("original genesis history root differs") + if mounted_genesis is None and (str(root) != anchor.binding.root or str(path) != str(Path(anchor.file("bootstrap").path).parent / GENESIS_NAME)): + raise OriginalEvidenceUnavailable("original genesis authority path differs") + if mounted_genesis is None: + _, created_roots, _ = _read_root_authority(Path(anchor.file("roots").path)) + if tuple(item for item in anchor.roots if item.role != "bootstrap") != created_roots: + raise OriginalEvidenceUnavailable("original genesis roots differ from creation authority") + for original_file in anchor.files: + if _file(original_file.role, Path(original_file.path)) != original_file: + raise OriginalEvidenceUnavailable("original genesis configuration or authorization changed") + for original_root in anchor.roots: + if identity(Path(original_root.host_path), directory=True) != (original_root.device, original_root.inode): + raise OriginalEvidenceUnavailable("original genesis protected root changed") + return anchor + + +def native_paths(anchor: GenesisAnchor, *, host: bool) -> dict[str, Path]: + state, journal = anchor.root("state"), anchor.root("journal") + state_path = Path(state.host_path if host else state.container_path) + journal_path = Path(journal.host_path if host else journal.container_path) + return {"child": state_path / "management-child.sqlite3", "handoff": state_path / "management-self-update.sqlite3", "recovery": state_path / "management-recovery.sqlite3", "operations": journal_path / "management-operations.sqlite3"} + + +def validate_native_genesis(anchor: GenesisAnchor, genesis_path: Path, *, host: bool) -> tuple[dict[str, object], dict[str, object]]: + """Check original native identities; an unknown outcome is not missing history.""" + state = Path(anchor.root("state").host_path if host else anchor.root("state").container_path) + encoded, _ = read_original(state / NATIVE_NAME) + native = json.loads(encoded) + expected_keys = {"profile", "history", "source", "genesis_sha256", "instance_id", "lifetime", "owner_fence", "stores"} + if not isinstance(native, dict) or set(native) != expected_keys or canonical(native) != encoded: + raise OriginalEvidenceUnavailable("native genesis original receipt is incomplete") + validate_original_lifetime(native["lifetime"]) + if (native["profile"], native["history"], native["source"], native["genesis_sha256"], native["instance_id"]) != (PROFILE, anchor.binding.history, anchor.binding.source, _file("genesis", genesis_path).sha256, anchor.instance_id): + raise OriginalEvidenceUnavailable("native genesis source/history identity differs") + if native["owner_fence"] != list(identity(state / ".supervisor-owner.lock")) or native["stores"] != {name: list(identity(path)) for name, path in native_paths(anchor, host=host).items()}: + raise OriginalEvidenceUnavailable("native genesis original owner/store inode differs") + launch_encoded, _ = read_original(state / LAUNCH_NAME) + launch = json.loads(launch_encoded) + if not isinstance(launch, dict) or set(launch) != {"profile", "history", "genesis_sha256", "native_sha256", "operation_id", "attempt_id", "lifetime"} or canonical(launch) != launch_encoded: + raise OriginalEvidenceUnavailable("original genesis first-launch boundary is incomplete") + if launch["profile"] != PROFILE or launch["history"] != anchor.binding.history or launch["genesis_sha256"] != native["genesis_sha256"] or launch["native_sha256"] != hashlib.sha256(encoded).hexdigest() or launch["lifetime"] != native["lifetime"]: + raise OriginalEvidenceUnavailable("original genesis first-launch boundary differs") + if not isinstance(launch["operation_id"], str) or not launch["operation_id"].startswith("bootstrap-") or str(uuid.UUID(launch["operation_id"][10:])) != launch["operation_id"][10:] or str(uuid.UUID(launch["attempt_id"])) != launch["attempt_id"]: + raise OriginalEvidenceUnavailable("original genesis first-launch identity differs") + return native, launch + + +def _validate_bootstrap_inputs(config: BootstrapConfig, bootstrap_path: Path) -> GenesisAnchor: + """Validate only original inputs while the caller holds history SH.""" + if config.original_genesis is None: + raise OriginalEvidenceUnavailable("ordinary bootstrap genesis selection missing") + anchor = validate_history_genesis(HISTORY_MOUNT, mounted_genesis=config.original_genesis) + if anchor.binding.source != source_identity() or anchor.instance_id != config.instance_id: + raise OriginalEvidenceUnavailable("ordinary bootstrap source or instance differs from genesis") + for root in anchor.roots: + if identity(Path(root.container_path), directory=True) != (root.device, root.inode): + raise OriginalEvidenceUnavailable("ordinary bootstrap original mounted root differs") + for role, path in (("bootstrap", bootstrap_path), ("management", Path("/bootstrap/management.yaml"))): + original = anchor.file(role) + actual = _file(role, path) + if (actual.device, actual.inode, actual.sha256) != (original.device, original.inode, original.sha256): + raise OriginalEvidenceUnavailable("ordinary bootstrap original configuration differs") + if (config.state_root, config.journal_root, config.oauth_root) != tuple(Path(anchor.root(role).container_path) for role in ("state", "journal", "oauth")): + raise OriginalEvidenceUnavailable("ordinary bootstrap retained paths differ from genesis") + _validate_management_settings(config, Path(anchor.root("bootstrap").container_path) / "management.yaml", bootstrap_path) + from .bootstrap_supervisor import _verify_fresh_console_entrypoint + + runtime = next(item for item in config.runtimes if item.reference == config.initial_runtime) + if not runtime.verify(initial=True): + raise OriginalEvidenceUnavailable("ordinary bootstrap original console receipt differs") + _verify_fresh_console_entrypoint(runtime) + return anchor + + +def validate_bootstrap_genesis(config: BootstrapConfig, bootstrap_path: Path) -> GenesisAnchor: + """Refuse missing/replaced originals before even a diagnostic directory.""" + with history_lock(HISTORY_MOUNT, exclusive=False): + return _validate_bootstrap_inputs(config, bootstrap_path) + + +def prepare_original_genesis(supervisor: ManagementBootstrapSupervisor, owner_start_token: str) -> tuple[str, str] | None: + """Called only by the actual ordinary bootstrap under its own exclusive lock.""" + from .fresh_host_observation import _assert_pid_flock + from .operation_store import ManagementOperationStore + from .recovery_mailbox import RecoveryMailbox + from .self_update_handoff import HandoffStore + + config = supervisor.config + if config.original_genesis is None or supervisor.bootstrap_config_path is None: + raise OriginalEvidenceUnavailable("ordinary bootstrap genesis selection missing") + descriptor = os.open(config.state_root / ".supervisor-owner.lock", os.O_RDONLY | os.O_NOFOLLOW) + try: + _assert_pid_flock(os.getpid(), descriptor, "WRITE") + finally: + os.close(descriptor) + with history_lock(HISTORY_MOUNT, exclusive=False): + anchor = _validate_bootstrap_inputs(config, supervisor.bootstrap_config_path) + native_path, launch_path = config.state_root / NATIVE_NAME, config.state_root / LAUNCH_NAME + if native_path.exists() or native_path.is_symlink() or launch_path.exists() or launch_path.is_symlink(): + validate_native_genesis(anchor, config.original_genesis, host=False) + return None + frontier, _ = read_original(HISTORY_MOUNT / "frontier.json") + if frontier != canonical({"count": 0, "digest": ""}) or {path.name for path in HISTORY_MOUNT.iterdir()} != {"history.lock", "authority.json", "frontier.json"}: + raise OriginalEvidenceUnavailable("ordinary genesis first launch has an advanced or partial history") + allowed = {".supervisor-owner.lock", "logs"} + if any(path.name not in allowed or path.is_symlink() for path in config.state_root.iterdir()) or any(config.journal_root.iterdir()) or any(config.oauth_root.iterdir()): + raise OriginalEvidenceUnavailable("ordinary genesis cannot initialize earlier or interrupted roots") + logs = config.state_root / "logs" + if logs.exists() and (not logs.is_dir() or logs.lstat().st_uid != os.geteuid()): + raise OriginalEvidenceUnavailable("ordinary genesis diagnostics directory differs") + previous_umask = os.umask(0o077) + try: + supervisor.store.publish_supervisor(supervisor.incarnation, os.getpid(), owner_start_token) + supervisor.store.require_existing = True + with closing(ManagementOperationStore(config.journal_root)._connect()): + pass + with closing(HandoffStore(config.state_root, config.instance_id)._connect()): + pass + RecoveryMailbox(config.state_root, config.instance_id).initialize() + for path in native_paths(anchor, host=False).values(): + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + _sync(config.state_root) + _sync(config.journal_root) + lifetime = _lifetime() + native = { + "profile": PROFILE, + "history": anchor.binding.history, + "source": anchor.binding.source, + "genesis_sha256": _file("genesis", config.original_genesis).sha256, + "instance_id": anchor.instance_id, + "lifetime": lifetime, + "owner_fence": identity(config.state_root / ".supervisor-owner.lock"), + "stores": {name: identity(path) for name, path in native_paths(anchor, host=False).items()}, + } + _create(config.state_root, NATIVE_NAME, native) + logger.bind(original_history=anchor.binding.root, original_boundary="genesis-stores", evidence_only=True).info("Original ordinary bootstrap stores captured: history={}", anchor.binding.history) + operation, attempt = "bootstrap-" + str(uuid.uuid4()), str(uuid.uuid4()) + _create(config.state_root, LAUNCH_NAME, {"profile": PROFILE, "history": anchor.binding.history, "genesis_sha256": native["genesis_sha256"], "native_sha256": hashlib.sha256(canonical(native)).hexdigest(), "operation_id": operation, "attempt_id": attempt, "lifetime": lifetime}) + validate_native_genesis(anchor, config.original_genesis, host=False) + logger.bind(original_history=anchor.binding.root, original_boundary="genesis-first-launch", evidence_only=True).info("Original ordinary bootstrap first-launch boundary recorded: history={}", anchor.binding.history) + return operation, attempt + finally: + os.umask(previous_umask) + + +def main() -> int: + from auto_coder.logger_config import setup_logger + + parser = argparse.ArgumentParser(description="Capture separately authorized ordinary genesis originals; no Docker effects or recovery authority") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--authorization", type=Path) + mode.add_argument("--prepare-roots", type=Path) + parser.add_argument("--log-file", required=True, type=Path) + args = parser.parse_args() + setup_logger(stream=sys.stdout, log_file=str(args.log_file)) + try: + if args.prepare_roots is not None: + print(json.dumps({"original_root_authority": str(prepare_original_genesis_roots(args.prepare_roots))}, sort_keys=True)) + else: + print(json.dumps(asdict(capture_original_genesis(args.authorization)), sort_keys=True)) + return 0 + except (OSError, ValueError, TypeError, KeyError, sqlite3.Error) as error: + parser.exit(1, "original ordinary genesis unavailable: " + str(error) + "\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/auto_coder/management/recovery_containment.py b/src/auto_coder/management/recovery_containment.py new file mode 100644 index 000000000..544e316df --- /dev/null +++ b/src/auto_coder/management/recovery_containment.py @@ -0,0 +1,562 @@ +"""Independently repeatable current ordinary-recovery containment observations. + +This read-only producer neither enrolls a scope, inhibits a controller, acquires +an owner fence, repairs history, creates admission nor grants recovery authority. +The fixed protected selection and every current technical observation are joined +here; the epoch issuer separately establishes complete original history and READY. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import select +import sqlite3 +import stat +import subprocess +import tarfile +import time +import uuid +from contextlib import ExitStack +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import cast + +from auto_coder.logger_config import get_logger + +from .archived_exclusions import unique_pairs +from .containment_enrollment import OriginalFile, _approved_directory, _file_identity, _protected, _trusted_ancestors +from .containment_kernel_subobservation import _holder_start_token, _require_capture_host, capture_original_caddy_container +from .containment_observation import DockerObservation, observe_recovery_docker +from .containment_route_manifest import RecoveryRouteScope, RecoveryRouteSelection, RouteManifestObservation, observe_recovery_route_manifest +from .docker_inspection import canonical_legacy_inspection +from .fresh_host import _evidence +from .fresh_host_observation import ( + PriorContainer, + _assert_pid_flock, + _capture_prior_kernel, + _capture_supervisor_identity, + _cgroup_tasks, + _engine_id, + _inspect, + assert_container_published_listener, + assert_host_listener_absent, + assert_prior_network_unreferenced, + assert_prior_processes_absent, +) +from .installed_bootstrap_mount_profile import _inspect as _full_inspect +from .installed_bootstrap_mount_profile import _mounts +from .original_attempts import source_identity +from .proxy_reconnect import ProxyBinding +from .recovery_containment_model import ( + RECOVERY_ENROLLMENT_ROOT, + NamedDescriptor, + RecoveryContainmentUnavailable, + RecoveryEnrollment, + RecoveryEnrollmentObservation, + RecoveryFile, + RecoveryOriginal, + RecoveryResources, + RecoveryTarget, +) +from .recovery_containment_profile import RecoveryManagerFact, _mapped_file, assert_disjoint_target, digest, read_bound_file, validate_manager_profile, validate_proxy_profile +from .recovery_containment_resources import RecoveryResourceObservation, _private_lock, hold_observed_resources + +logger = get_logger(__name__) +_REPORT_SCHEMA = "current-recovery-containment-v1" +_STATUS = "observed-current-recovery-containment" +_MAX_BYTES = 8 * 1024 * 1024 + + +@dataclass(frozen=True) +class RecoveryKernelFact: + generation_id: str + current_state: str + boot_id: str + manager_pid: int + manager_start_token: str + proxy_pid: int + proxy_start_token: str + original_namespace: str + owner_device: int + owner_inode: int + published_port: int + original_cgroups_empty: bool + original_namespace_unoccupied: bool + original_listener_absent: bool + + +@dataclass(frozen=True) +class RecoveryTargetFact: + container_id: str + pid: int + start_token: str + boot_id: str + network_namespace: str + pid_namespace: str + cgroup_path: str + cgroup_device: int + cgroup_inode: int + owner_device: int + owner_inode: int + published_port: int + + +@dataclass(frozen=True) +class RecoveryWatcherFact: + unit: str + manager_id: str + proxy_id: str + configuration: RecoveryFile + source: RecoveryFile + lock_device: int + lock_inode: int + + +@dataclass(frozen=True) +class RecoveryContainmentFacts: + enrollment: RecoveryEnrollmentObservation + source_identity: str + docker: tuple[DockerObservation, ...] + managers: tuple[RecoveryManagerFact, ...] + proxy_inspections: tuple[tuple[str, str], ...] + kernel: tuple[RecoveryKernelFact, ...] + target: RecoveryTargetFact + resources: RecoveryResourceObservation + routes: RouteManifestObservation + watchers: tuple[RecoveryWatcherFact, ...] + effective_files: tuple[RecoveryFile, ...] + + +@dataclass(frozen=True) +class RecoveryContainmentReport: + schema: str + status: str + observation_id: str + observer_pid: int + observer_start_token: str + boot_id: str + started_monotonic_ns: int + finished_monotonic_ns: int + facts: RecoveryContainmentFacts + evidence_sha256: str + grant_issued: bool = False + history_asserted: bool = False + admission_created: bool = False + + @property + def managers(self) -> tuple[RecoveryManagerFact, ...]: + return self.facts.managers + + @property + def docker_ids(self) -> tuple[str, ...]: + return tuple(item.container_id for item in self.facts.docker) + + @property + def target_pid(self) -> int: + return self.facts.target.pid + + @property + def target_start_token(self) -> str: + return self.facts.target.start_token + + @property + def enrollment(self) -> RecoveryEnrollmentObservation: + return self.facts.enrollment + + +def _load_enrollment(enrollment_id: str, *, root: Path, expected_uid: int) -> RecoveryEnrollmentObservation: + """Controlled protected fixture seam; the production factory fixes both.""" + from .recovery_containment_model import _IDENTIFIER + + if not isinstance(enrollment_id, str) or not _IDENTIFIER.fullmatch(enrollment_id) or os.geteuid() != expected_uid: + raise RecoveryContainmentUnavailable("invalid protected recovery enrollment selector") + _approved_directory(root, uid=expected_uid) + path = root / (enrollment_id + ".json") + content, info = _protected(path, uid=expected_uid, size_limit=1024 * 1024) + if info.st_nlink != 1: + raise RecoveryContainmentUnavailable("recovery enrollment has an ambiguous file identity") + selected = RecoveryEnrollment.parse(json.loads(content, object_pairs_hook=unique_pairs)) + if selected.enrollment_id != enrollment_id: + raise RecoveryContainmentUnavailable("protected recovery enrollment selector differs") + return RecoveryEnrollmentObservation(selected, hashlib.sha256(content).hexdigest(), info.st_dev, info.st_ino, info.st_ctime_ns) + + +def load_enrollment(enrollment_id: str) -> RecoveryEnrollmentObservation: + """Load only the fixed trusted-host scope; no request/environment override.""" + _require_capture_host() + return _load_enrollment(enrollment_id, root=RECOVERY_ENROLLMENT_ROOT, expected_uid=0) + + +def route_selection(approved: RecoveryEnrollmentObservation) -> RecoveryRouteSelection: + """Pass the full independently protected recovery scope to shared readers.""" + selected = approved.enrollment + originals: list[OriginalFile] = [] + for item in selected.originals: + originals.extend((OriginalFile("bootstrap-config", **asdict(item.bootstrap_config)), OriginalFile("management-config", **asdict(item.management_config)), OriginalFile("proxy-config", **asdict(item.proxy_config)))) + originals.extend((OriginalFile("bootstrap-config", **asdict(selected.target.bootstrap_config)), OriginalFile("management-config", **asdict(selected.target.management_config)))) + for watcher in selected.watchers: + originals.extend((OriginalFile("watcher-source", **asdict(watcher.source)), OriginalFile("watcher-config", **asdict(watcher.configuration)))) + scope = RecoveryRouteScope(selected.deployment_id, selected.enrollment_id, selected.authorization_id, selected.profile, selected.container_ids, tuple(originals)) + return RecoveryRouteSelection(scope, approved.sha256) + + +def _engine(enrollment: RecoveryEnrollment) -> tuple[DockerObservation, ...]: + managers = tuple(item.management_id for item in enrollment.originals) + (enrollment.target.management_id,) + proxies = tuple(item.proxy_id for item in enrollment.originals) + docker = observe_recovery_docker(managers, proxies) + selected = set(enrollment.container_ids) + if any(item.engine_id != enrollment.engine_id or item.management_route_roles and item.container_id not in selected for item in docker): + raise RecoveryContainmentUnavailable("foreign, unknown or omitted recovery management candidate") + if any(item.container_id in selected and (item.restart_policy != "no" or item.maximum_retry_count != 0) for item in docker): + raise RecoveryContainmentUnavailable("an ordinary recovery Docker restart route remains enabled") + return docker + + +def _original_kernel(original: RecoveryOriginal, manager: RecoveryManagerFact, current: dict[str, DockerObservation]) -> RecoveryKernelFact: + if original.state == "never-started": + for identifier in (original.management_id, original.proxy_id): + item = current[identifier] + if item.running or item.status != "created" or item.pid != 0 or item.restart_count != 0 or item.started_at != "0001-01-01T00:00:00Z" or item.finished_at != "0001-01-01T00:00:00Z": + raise RecoveryContainmentUnavailable("selected never-started Docker object has a lifetime") + assert_host_listener_absent(manager.published_port) + return RecoveryKernelFact(original.generation_id, "currently-created-never-started", "", 0, "", 0, "", "", 0, 0, manager.published_port, False, False, True) + if original.prior_evidence is None or original.owner is None or original.namespace_pin is None: + raise RecoveryContainmentUnavailable("original lifetime evidence or resource identity is missing") + read_bound_file(original.prior_evidence) + prior, proxy, prior_sha = _evidence(Path(original.prior_evidence.path)) + if prior_sha != original.prior_evidence.sha256 or original.boot_id != Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip(): + raise RecoveryContainmentUnavailable("original evidence differs or came from another boot") + if ( + prior.engine_id != proxy.engine_id + or prior.engine_id != current[original.management_id].engine_id + or (prior.container_id, prior.image_id, prior.instance_id) != (original.management_id, original.management_image_id, original.instance_id) + or (proxy.container_id, proxy.image_id, proxy.network_mode) != (original.proxy_id, original.proxy_image_id, "container:" + original.management_id) + or prior.net_namespace != proxy.net_namespace + or prior.net_namespace != original.namespace_pin.namespace + or (prior.owner_lock_device, prior.owner_lock_inode) != (original.owner.device, original.owner.inode) + or (prior.bootstrap_source, prior.bootstrap_sha256, prior.published_port) != (original.bootstrap_config.path, original.bootstrap_config.sha256, manager.published_port) + or prior.oauth_source != manager.oauth_root + ): + raise RecoveryContainmentUnavailable("recovery captured original scope differs") + oauth = Path(manager.oauth_root).stat() + if (oauth.st_dev, oauth.st_ino) != (prior.oauth_device, prior.oauth_inode): + raise RecoveryContainmentUnavailable("recovery captured OAuth object was replaced") + for lifetime in (prior, proxy): + observed = current[lifetime.container_id] + mounts = tuple(sorted((item.source, item.destination, item.writable) for item in original.mounts if item.container_id == lifetime.container_id)) + mount_sha = hashlib.sha256(json.dumps(mounts, separators=(",", ":")).encode()).hexdigest() + if observed.running or observed.pid != 0 or observed.status != "exited" or observed.image_id != lifetime.image_id or observed.started_at != lifetime.started_at or observed.restart_count != lifetime.restart_count or mount_sha != lifetime.mounts_sha256: + raise RecoveryContainmentUnavailable("recovery original Docker lifetime/configuration changed") + assert_prior_processes_absent(lifetime) + assert_prior_network_unreferenced(prior.net_namespace) + assert_host_listener_absent(prior.published_port) + read_bound_file(original.prior_evidence) + return RecoveryKernelFact(original.generation_id, "original-lifetimes-absent", original.boot_id, prior.pid, prior.start_token, proxy.pid, proxy.start_token, prior.net_namespace, original.owner.device, original.owner.inode, prior.published_port, True, True, True) + + +def _target_kernel(enrollment: RecoveryEnrollment, current: dict[str, DockerObservation], kernels: tuple[RecoveryKernelFact, ...]) -> RecoveryTargetFact: + target = enrollment.target + observed = current[target.management_id] + if not observed.running or observed.status != "running" or observed.pid <= 1 or observed.image_id != target.management_image_id: + raise RecoveryContainmentUnavailable("exact recovery target is not running its custody entrypoint") + proc = Path("/proc") / str(observed.pid) + with ExitStack() as stack: + lifetime = os.pidfd_open(observed.pid, 0) + stack.callback(os.close, lifetime) + poller = select.poll() + poller.register(lifetime, select.POLLIN) + token = _holder_start_token(observed.pid) + network, pid_namespace = os.readlink(proc / "ns/net"), os.readlink(proc / "ns/pid") + if network in {item.original_namespace for item in kernels} or network == os.readlink("/proc/1/ns/net") or pid_namespace == os.readlink("/proc/1/ns/pid"): + raise RecoveryContainmentUnavailable("recovery target namespace aliases an original or the host") + command = (proc / "cmdline").read_bytes().rstrip(b"\0").split(b"\0") + if not command or command[0] not in (b"python", b"/opt/auto-coder/.venv/bin/python") or command[1:] != [b"-m", b"auto_coder.management.bootstrap_supervisor", b"--config", b"/bootstrap/config.json", b"--recovery-epoch"]: + raise RecoveryContainmentUnavailable("target process is not the exact prelaunch consumer") + expected_cgroup = "/system.slice/docker-" + target.management_id + ".scope" + if (proc / "cgroup").read_text(encoding="ascii").splitlines() != ["0::" + expected_cgroup]: + raise RecoveryContainmentUnavailable("recovery target cgroup differs") + cgroup = Path("/sys/fs/cgroup") / expected_cgroup.lstrip("/") + info = cgroup.stat() + if _cgroup_tasks(cgroup) != {observed.pid}: + raise RecoveryContainmentUnavailable("recovery prelaunch target has unexpected child tasks") + _private_lock(Path(target.owner.path), target.owner, uid=0) + owner = os.open(target.owner.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + stack.callback(os.close, owner) + if (os.fstat(owner).st_dev, os.fstat(owner).st_ino) != (target.owner.device, target.owner.inode): + raise RecoveryContainmentUnavailable("recovery target owner changed during open") + _assert_pid_flock(observed.pid, owner, "WRITE") + assert_container_published_listener(target.management_id, target.management_image_id, target.published_port) + if _holder_start_token(observed.pid) != token or poller.poll(0) or os.readlink(proc / "ns/net") != network or _cgroup_tasks(cgroup) != {observed.pid} or (cgroup.stat().st_dev, cgroup.stat().st_ino) != (info.st_dev, info.st_ino): + raise RecoveryContainmentUnavailable("recovery target lifetime changed during observation") + return RecoveryTargetFact(target.management_id, observed.pid, token, Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip(), network, pid_namespace, expected_cgroup, info.st_dev, info.st_ino, target.owner.device, target.owner.inode, target.published_port) + + +def _watchers(enrollment: RecoveryEnrollment, routes: RouteManifestObservation) -> tuple[RecoveryWatcherFact, ...]: + from . import proxy_watch_cli + + actual_source = str(Path(cast(str, proxy_watch_cli.__file__)).resolve(strict=True)) + configs = {item.configuration.path for item in enrollment.watchers} + selected_units = {item.unit for item in enrollment.watchers} + routes_by_name = {item.unit: item for item in routes.systemd} + result = [] + pairs = {(item.management_id, item.management_image_id, item.proxy_id, item.proxy_image_id): item for item in enrollment.originals} + for selected in enrollment.watchers: + if selected.source.path != actual_source: + raise RecoveryContainmentUnavailable("watcher source is not the actual current installed module") + read_bound_file(selected.source, source=True) + content = read_bound_file(selected.configuration) + raw = json.loads(content, object_pairs_hook=unique_pairs) + if not isinstance(raw, dict) or set(raw) != {"state_root", "manager_id", "manager_image", "proxy_id", "proxy_image", "config_sha256"} or any(not isinstance(value, str) for value in raw.values()): + raise RecoveryContainmentUnavailable("watcher effective configuration differs") + binding = ProxyBinding(raw["manager_id"], raw["manager_image"], raw["proxy_id"], raw["proxy_image"], raw["config_sha256"]) + binding.validate() + pair = pairs.get((binding.manager_id, binding.manager_image, binding.proxy_id, binding.proxy_image)) + if pair is None or pair.proxy_config.sha256 != binding.config_sha256 or selected.lock.path != str(Path(raw["state_root"]) / "proxy-watch.lock"): + raise RecoveryContainmentUnavailable("watcher source/configuration does not use the exact enrolled controller lock") + root = Path(raw["state_root"]) + _approved_directory(root, uid=0) + fact = routes_by_name.get(selected.unit) + if fact is None or not fact.inhibited or fact.main_pid != 0 or {selected.source.path, selected.configuration.path} - {item.path for item in fact.command_sources}: + raise RecoveryContainmentUnavailable("watcher installed command or technical inhibition differs") + result.append(RecoveryWatcherFact(selected.unit, binding.manager_id, binding.proxy_id, selected.configuration, selected.source, selected.lock.device, selected.lock.inode)) + source_sets = [(item.unit, item.command_sources) for item in routes.systemd] + source_sets += [(str(item.uid) + ":" + item.fact.unit, item.fact.command_sources) for item in routes.user_systemd] + source_sets += [(item.path, item.command_sources) for item in routes.cron_entries] + for name, sources in source_sets: + paths = {item.path for item in sources} + if actual_source in paths and (len(paths & configs) != 1 or name.endswith(".service") and name not in selected_units): + raise RecoveryContainmentUnavailable("unknown watcher invocation/configuration is outside the actual controller fences") + return tuple(result) + + +def _effective_files(enrollment: RecoveryEnrollment, managers: tuple[RecoveryManagerFact, ...]) -> tuple[RecoveryFile, ...]: + result: dict[str, RecoveryFile] = {} + members: tuple[RecoveryOriginal | RecoveryTarget, ...] = (*enrollment.originals, enrollment.target) + for selected, manager in zip(members, managers): + raw = json.loads(read_bound_file(selected.bootstrap_config), object_pairs_hook=unique_pairs) + for name in ("original_genesis", "recovery_epoch_config"): + configured = raw.get(name) + if configured is None: + continue + path = Path(_mapped_file(configured, manager.mounts, writable=False)) + data, info = _protected(path, uid=0, size_limit=1024 * 1024) + if info.st_nlink != 1: + raise RecoveryContainmentUnavailable("effective original/consumer input is aliased") + result[str(path)] = RecoveryFile(str(path), hashlib.sha256(data).hexdigest(), info.st_dev, info.st_ino) + if name == "recovery_epoch_config": + _consumer_custody_profile(enrollment, data) + return tuple(result[key] for key in sorted(result)) + + +def _consumer_custody_profile(enrollment: RecoveryEnrollment, encoded: bytes) -> None: + """Bind the target's actual prelaunch inputs to this exact selected scope. + + History completeness, epoch EX and grant admission remain issuer duties. + Here those input roles must still name their real mapped objects rather + than an alternate socket, a different original owner or missing resources. + """ + from .recovery_epoch_consumer import parse_consumer_configuration + + selected = parse_consumer_configuration(encoded) + target = enrollment.target + mounts = {item.destination: item for item in target.mounts} + socket_directory = mounts.get("/management-recovery") + if ( + selected.authorization_id != enrollment.authorization_id + or selected.issuer_uid != 0 + or socket_directory is None + or socket_directory.source != "/var/lib/auto-coder/recovery-epoch-peer" + or socket_directory.writable + or not stat.S_ISDIR(socket_directory.mode) + or (selected.socket_directory_device, selected.socket_directory_inode) != (socket_directory.device, socket_directory.inode) + or (selected.target_owner.name, selected.target_owner.device, selected.target_owner.inode) != ("target-owner", target.owner.device, target.owner.inode) + ): + raise RecoveryContainmentUnavailable("recovery consumer's actual issuer socket or target owner differs") + owners = {item.name: item for item in selected.original_fences} + expected = {"history"} + allowed_owner_destinations = set() + transferred = {item.name: item for item in selected.transferred_resources} + expected_transfers = {"epoch"} + for index, original in enumerate(enrollment.originals): + if original.owner is None: + continue + name, destination = f"owner-{index}", f"/management-recovery-originals/{index}/owner.lock" + expected.add(name) + allowed_owner_destinations.add(destination) + chosen, mount = owners.get(name), mounts.get(destination) + if ( + chosen is None + or mount is None + or mount.writable + or not stat.S_ISREG(mount.mode) + or chosen.path != destination + or mount.source != original.owner.path + or (chosen.device, chosen.inode) != (original.owner.device, original.owner.inode) + or (mount.device, mount.inode) != (chosen.device, chosen.inode) + ): + raise RecoveryContainmentUnavailable("recovery consumer original SH input is missing or bound to another owner") + namespace = transferred.get(f"namespace-{index}") + pin = original.namespace_pin + if namespace is None or pin is None or namespace.kind != "network-namespace" or (namespace.device, namespace.inode) != (pin.device, pin.inode): + raise RecoveryContainmentUnavailable("recovery consumer original namespace input differs") + expected_transfers.add(namespace.name) + if {item.destination for item in target.mounts if item.destination.startswith("/management-recovery-originals/")} != allowed_owner_destinations: + raise RecoveryContainmentUnavailable("recovery target has undeclared original owner mounts") + history = owners.get("history") + history_mount = mounts.get("/management-recovery-history/history.lock") + if set(owners) != expected or history is None or history_mount is None or history.path != history_mount.destination or history_mount.writable or not stat.S_ISREG(history_mount.mode) or (history.device, history.inode) != (history_mount.device, history_mount.inode): + raise RecoveryContainmentUnavailable("recovery consumer history SH input or original set differs") + for index, watcher in enumerate(enrollment.watchers): + controller = transferred.get(f"controller-{index}") + if controller is None or controller.kind != "transferred-ex" or (controller.device, controller.inode) != (watcher.lock.device, watcher.lock.inode): + raise RecoveryContainmentUnavailable("recovery consumer controller custody input differs") + expected_transfers.add(controller.name) + epoch = transferred.get("epoch") + if set(transferred) != expected_transfers or epoch is None or epoch.kind != "transferred-ex": + raise RecoveryContainmentUnavailable("recovery consumer deployment custody input is incomplete") + + +def _facts(approved: RecoveryEnrollmentObservation, resources: RecoveryResourceObservation) -> RecoveryContainmentFacts: + enrollment = approved.enrollment + docker = _engine(enrollment) + current = {item.container_id: item for item in docker} + full = {identifier: _full_inspect(identifier) for identifier in enrollment.container_ids} + if any(digest([full[identifier]]) != current[identifier].inspection_sha256 for identifier in full): + raise RecoveryContainmentUnavailable("recovery Docker objects changed after whole-engine enumeration") + members: tuple[RecoveryOriginal | RecoveryTarget, ...] = (*enrollment.originals, enrollment.target) + managers = tuple(validate_manager_profile(item, full[item.management_id], source=enrollment.source_identity) for item in members) + assert_disjoint_target(enrollment, managers) + proxies = tuple((item.generation_id, validate_proxy_profile(item, full[item.proxy_id])) for item in enrollment.originals) + kernels = tuple(_original_kernel(item, manager, current) for item, manager in zip(enrollment.originals, managers)) + target = _target_kernel(enrollment, current, kernels) + routes = observe_recovery_route_manifest(route_selection(approved), docker) + watchers = _watchers(enrollment, routes) + effective = _effective_files(enrollment, managers) + if not routes.closed_profile or routes.deployment_roots != tuple(sorted((item for original in members for item in original.mounts), key=lambda item: (item.container_id, item.destination))): + raise RecoveryContainmentUnavailable("recovery route closure omitted or changed an actual selected root") + # The complete route reader already repeats its system/user/cron catalog. + # Repeat the other actual predicates around that reader while custody is + # retained, without redundantly multiplying the full systemd scan count. + if ( + tuple(validate_manager_profile(item, full[item.management_id], source=enrollment.source_identity) for item in members) != managers + or tuple((item.generation_id, validate_proxy_profile(item, full[item.proxy_id])) for item in enrollment.originals) != proxies + or tuple(_original_kernel(item, manager, current) for item, manager in zip(enrollment.originals, managers)) != kernels + or _target_kernel(enrollment, current, kernels) != target + or _watchers(enrollment, routes) != watchers + or _effective_files(enrollment, managers) != effective + ): + raise RecoveryContainmentUnavailable("recovery source, kernel or effective configuration changed during observation") + if _engine(enrollment) != docker or any(_full_inspect(identifier) != full[identifier] for identifier in full): + raise RecoveryContainmentUnavailable("recovery complete engine/source/configuration changed during observation") + return RecoveryContainmentFacts(approved, enrollment.source_identity, docker, managers, proxies, kernels, target, resources, routes, watchers, effective) + + +def observe(enrollment_id: str, resources: RecoveryResources) -> RecoveryContainmentReport: + """Produce complete current facts or raise; partial evidence is never success.""" + try: + started = time.monotonic_ns() + approved = load_enrollment(enrollment_id) + resources = RecoveryResources.parse(json.loads(json.dumps(asdict(resources)))) + if approved.enrollment.source_identity != source_identity(): + raise RecoveryContainmentUnavailable("enrolled recovery package source is not the current observer source") + with hold_observed_resources(approved.enrollment, resources) as held: + last = _facts(approved, held) + if load_enrollment(enrollment_id) != approved: + raise RecoveryContainmentUnavailable("recovery protected enrollment changed during observations") + payload = digest(asdict(last)) + report = RecoveryContainmentReport(_REPORT_SCHEMA, _STATUS, str(uuid.uuid4()), os.getpid(), _holder_start_token(os.getpid()), resources.holder_boot_id, started, time.monotonic_ns(), last, payload) + logger.bind(observation_only=True, recovery_grant=False, admission_created=False).info("Current recovery containment observed: enrollment={} evidence={}", enrollment_id, payload) + return report + except (OSError, ValueError, TypeError, KeyError, IndexError, UnicodeError, sqlite3.Error, subprocess.SubprocessError, tarfile.TarError) as error: + if isinstance(error, RecoveryContainmentUnavailable): + raise + raise RecoveryContainmentUnavailable("current recovery containment unavailable: " + str(error)) from error + + +def verify(enrollment_id: str, previous: RecoveryContainmentReport | object, resources: RecoveryResources) -> RecoveryContainmentReport: + """A separate consumer repeats current reads; a saved status is no authority.""" + prior = asdict(previous) if isinstance(previous, RecoveryContainmentReport) else previous + if not isinstance(prior, dict) or set(prior) != set(RecoveryContainmentReport.__dataclass_fields__) or prior.get("schema") != _REPORT_SCHEMA or prior.get("status") != _STATUS or any(prior.get(name) is not False for name in ("grant_issued", "history_asserted", "admission_created")): + raise RecoveryContainmentUnavailable("previous recovery report is forged, partial or authorizing") + if any(type(prior.get(name)) is not int or cast(int, prior[name]) <= 0 for name in ("observer_pid", "started_monotonic_ns", "finished_monotonic_ns")) or not isinstance(prior.get("observer_start_token"), str): + raise RecoveryContainmentUnavailable("previous recovery observer lifetime differs") + if not isinstance(prior.get("facts"), dict) or prior.get("evidence_sha256") != digest(prior["facts"]): + raise RecoveryContainmentUnavailable("previous recovery fact digest differs") + if not isinstance(prior.get("observation_id"), str) or str(uuid.UUID(prior["observation_id"])) != prior["observation_id"]: + raise RecoveryContainmentUnavailable("previous recovery observation identity differs") + # The sampling command may have exited. Its PID is attribution only; the + # actual resource producers and holders are independently pinned and checked + # again by observe(), and a saved positive status supplies no authority. + current = observe(enrollment_id, resources) + if prior["boot_id"] != current.boot_id or not cast(int, prior["started_monotonic_ns"]) <= cast(int, prior["finished_monotonic_ns"]) <= current.started_monotonic_ns or current.started_monotonic_ns - cast(int, prior["finished_monotonic_ns"]) > 600 * 1_000_000_000: + raise RecoveryContainmentUnavailable("previous recovery sample is stale, future or from another boot") + if prior["evidence_sha256"] != current.evidence_sha256 or json.loads(json.dumps(prior["facts"])) != json.loads(json.dumps(asdict(current.facts))): + raise RecoveryContainmentUnavailable("previous recovery scope/source/resources/current facts differ") + return current + + +def capture_recovery_original_pair(original: RecoveryOriginal, *, source: str) -> tuple[str, PriorContainer, PriorContainer]: + """Capture real live original lifetimes without claiming a branch succeeded. + + This reads the separately selected command/configuration and the native + supervisor owner/presence. The caller must pin its real network namespace + while alive and install approval separately; this function does neither. + """ + _require_capture_host() + engine = _engine_id() + full = _full_inspect(original.management_id) + validate_manager_profile(original, full, source=source) + proxy_full = _full_inspect(original.proxy_id) + validate_proxy_profile(original, proxy_full) + values = canonical_legacy_inspection(_inspect(original.management_id)) + if values[1:3] != [original.management_image_id, True] or values[4] != "running": + raise RecoveryContainmentUnavailable("recovery original is not the exact live manager") + mounts = _mounts(full) + manager = _capture_prior_kernel(original.management_id, original.management_image_id, engine, values, mounts, require_owner_lock=True) + manager = _capture_supervisor_identity(manager, values, mounts, Path(original.bootstrap_config.path)) + proxy = capture_original_caddy_container(original.proxy_id, original.proxy_image_id, expected_mounts=_mounts(proxy_full)) + if _full_inspect(original.management_id) != full or _full_inspect(original.proxy_id) != proxy_full or _engine_id() != engine or manager.net_namespace != proxy.net_namespace: + raise RecoveryContainmentUnavailable("recovery original changed during live capture") + validate_manager_profile(original, full, source=source) + return Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip(), manager, proxy + + +def _input(path: Path) -> object: + if not path.is_absolute() or path != path.resolve(strict=True): + raise RecoveryContainmentUnavailable("recovery observation input is not canonical") + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + first = os.fstat(stream.fileno()) + if not stat.S_ISREG(first.st_mode): + raise RecoveryContainmentUnavailable("recovery observation input is not regular") + data = stream.read(_MAX_BYTES + 1) + if len(data) > _MAX_BYTES or _file_identity(first) != _file_identity(os.fstat(stream.fileno())) or _file_identity(first) != _file_identity(path.lstat()): + raise RecoveryContainmentUnavailable("recovery observation input changed or exceeds its bound") + return json.loads(data, object_pairs_hook=unique_pairs) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Read current ordinary recovery containment; no admission or grant") + parser.add_argument("action", choices=("observe", "verify")) + parser.add_argument("--enrollment", required=True) + parser.add_argument("--resources", required=True, type=Path) + parser.add_argument("--previous", type=Path) + args = parser.parse_args() + try: + resources = RecoveryResources.parse(_input(args.resources)) + if (args.action == "verify") != (args.previous is not None): + raise RecoveryContainmentUnavailable("separate verification requires exactly one previous report") + report = verify(args.enrollment, _input(args.previous), resources) if args.previous is not None else observe(args.enrollment, resources) + print(json.dumps(asdict(report), sort_keys=True, separators=(",", ":")), flush=True) + return 0 + except (OSError, ValueError, TypeError, KeyError, IndexError, UnicodeError, subprocess.SubprocessError) as error: + logger.bind(observation_only=True, recovery_grant=False, admission_created=False).warning("Current recovery containment unavailable: {}", str(error)) + print(json.dumps({"schema": _REPORT_SCHEMA, "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": str(error)}, sort_keys=True), flush=True) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/auto_coder/management/recovery_containment_model.py b/src/auto_coder/management/recovery_containment_model.py new file mode 100644 index 000000000..100145b5e --- /dev/null +++ b/src/auto_coder/management/recovery_containment_model.py @@ -0,0 +1,309 @@ +"""Finite scope and resource types for observation-only ordinary recovery. + +Protected enrollment selects the independently inspectable objects. None of these +records attests original history, technical containment, admission or a grant. +""" + +from __future__ import annotations + +import re +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from .containment_enrollment import OriginalNetworkPin +from .containment_route_manifest import DeploymentRootBinding + +RECOVERY_ENROLLMENT_ROOT = Path("/var/lib/auto-coder/approved-recovery-containment-enrollments") +RECOVERY_PROFILE = "ordinary-recovery-v1" +_MAX_GENERATIONS = 16 +_HASH = re.compile(r"[0-9a-f]{64}") +_IMAGE = re.compile(r"sha256:[0-9a-f]{64}") +_IDENTIFIER = re.compile(r"[A-Za-z0-9_.:-]{1,128}") + + +class RecoveryContainmentUnavailable(ValueError): + """A required current recovery observation is absent or unsupported.""" + + +def _object(value: object, fields: object, name: str) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != set(cast(dict[str, object], fields)): + raise RecoveryContainmentUnavailable(name + " schema differs") + return value + + +def _text(value: object, pattern: re.Pattern[str], name: str) -> str: + if not isinstance(value, str) or not pattern.fullmatch(value): + raise RecoveryContainmentUnavailable(name + " differs") + return value + + +def _path(value: object) -> str: + if not isinstance(value, str) or not value or len(value) > 4096 or not Path(value).is_absolute() or str(Path(value)) != value or ".." in Path(value).parts or value == "/": + raise RecoveryContainmentUnavailable("recovery path is not canonical") + return value + + +def _positive(value: object) -> int: + if type(value) is not int or value <= 0: + raise RecoveryContainmentUnavailable("recovery kernel identity differs") + return value + + +def _boot(value: object) -> str: + try: + if not isinstance(value, str) or str(uuid.UUID(value)) != value: + raise ValueError("noncanonical boot identity") + except (ValueError, TypeError, AttributeError) as error: + raise RecoveryContainmentUnavailable("recovery boot identity differs") from error + return value + + +@dataclass(frozen=True) +class RecoveryFile: + path: str + sha256: str + device: int + inode: int + + @classmethod + def parse(cls, value: object) -> RecoveryFile: + raw = _object(value, cls.__dataclass_fields__, "recovery file") + return cls(_path(raw["path"]), _text(raw["sha256"], _HASH, "file digest"), _positive(raw["device"]), _positive(raw["inode"])) + + +@dataclass(frozen=True) +class RecoveryLock: + path: str + device: int + inode: int + + @classmethod + def parse(cls, value: object) -> RecoveryLock: + raw = _object(value, cls.__dataclass_fields__, "recovery lock") + return cls(_path(raw["path"]), _positive(raw["device"]), _positive(raw["inode"])) + + +def _mounts(value: object, containers: tuple[str, ...]) -> tuple[DeploymentRootBinding, ...]: + if not isinstance(value, list) or not 1 <= len(value) <= 64: + raise RecoveryContainmentUnavailable("recovery bind inventory is absent or oversized") + mounts = tuple(DeploymentRootBinding.parse(item) for item in value) + keys = tuple((item.container_id, item.destination) for item in mounts) + if keys != tuple(sorted(set(keys))) or {item.container_id for item in mounts} != set(containers): + raise RecoveryContainmentUnavailable("recovery bind inventory is ambiguous or foreign") + for item in mounts: + _path(item.source) + _path(item.destination) + return mounts + + +@dataclass(frozen=True) +class RecoveryOriginal: + generation_id: str + instance_id: str + management_id: str + management_image_id: str + proxy_id: str + proxy_image_id: str + state: str + boot_id: str + bootstrap_config: RecoveryFile + management_config: RecoveryFile + proxy_config: RecoveryFile + mounts: tuple[DeploymentRootBinding, ...] + owner: RecoveryLock | None + prior_evidence: RecoveryFile | None + namespace_pin: OriginalNetworkPin | None + + @classmethod + def parse(cls, value: object) -> RecoveryOriginal: + raw = _object(value, cls.__dataclass_fields__, "recovery original") + manager = _text(raw["management_id"], _HASH, "original manager") + proxy = _text(raw["proxy_id"], _HASH, "original proxy") + if manager == proxy: + raise RecoveryContainmentUnavailable("original manager and proxy alias") + state = raw["state"] + if state not in ("started", "never-started"): + raise RecoveryContainmentUnavailable("original current-state selection differs") + if state == "started": + boot = _boot(raw["boot_id"]) + owner = RecoveryLock.parse(raw["owner"]) + prior = RecoveryFile.parse(raw["prior_evidence"]) + pin = OriginalNetworkPin.parse(raw["namespace_pin"]) + if pin.holder_boot_id != boot: + raise RecoveryContainmentUnavailable("original capture and namespace boot differ") + else: + if raw["boot_id"] != "" or any(raw[name] is not None for name in ("owner", "prior_evidence", "namespace_pin")): + raise RecoveryContainmentUnavailable("never-started object cannot invent old resources") + boot, owner, prior, pin = "", None, None, None + return cls( + _text(raw["generation_id"], _IDENTIFIER, "generation identity"), + _text(raw["instance_id"], _IDENTIFIER, "original instance identity"), + manager, + _text(raw["management_image_id"], _IMAGE, "original manager image"), + proxy, + _text(raw["proxy_image_id"], _IMAGE, "original proxy image"), + str(state), + boot, + RecoveryFile.parse(raw["bootstrap_config"]), + RecoveryFile.parse(raw["management_config"]), + RecoveryFile.parse(raw["proxy_config"]), + _mounts(raw["mounts"], (manager, proxy)), + owner, + prior, + pin, + ) + + +@dataclass(frozen=True) +class RecoveryTarget: + instance_id: str + management_id: str + management_image_id: str + bootstrap_config: RecoveryFile + management_config: RecoveryFile + mounts: tuple[DeploymentRootBinding, ...] + owner: RecoveryLock + published_port: int + + @classmethod + def parse(cls, value: object) -> RecoveryTarget: + raw = _object(value, cls.__dataclass_fields__, "recovery target") + manager = _text(raw["management_id"], _HASH, "target manager") + port = raw["published_port"] + if type(port) is not int or not 0 < port <= 65535: + raise RecoveryContainmentUnavailable("target published port differs") + return cls( + _text(raw["instance_id"], _IDENTIFIER, "target instance"), + manager, + _text(raw["management_image_id"], _IMAGE, "target image"), + RecoveryFile.parse(raw["bootstrap_config"]), + RecoveryFile.parse(raw["management_config"]), + _mounts(raw["mounts"], (manager,)), + RecoveryLock.parse(raw["owner"]), + port, + ) + + +@dataclass(frozen=True) +class RecoveryWatcher: + unit: str + configuration: RecoveryFile + source: RecoveryFile + lock: RecoveryLock + + @classmethod + def parse(cls, value: object) -> RecoveryWatcher: + raw = _object(value, cls.__dataclass_fields__, "recovery watcher") + return cls(_text(raw["unit"], re.compile(r"[A-Za-z0-9_.@:-]{1,120}\.service"), "watcher unit"), RecoveryFile.parse(raw["configuration"]), RecoveryFile.parse(raw["source"]), RecoveryLock.parse(raw["lock"])) + + +@dataclass(frozen=True) +class RecoveryEnrollment: + schema: str + deployment_id: str + enrollment_id: str + authorization_id: str + profile: str + engine_id: str + source_identity: str + originals: tuple[RecoveryOriginal, ...] + target: RecoveryTarget + watchers: tuple[RecoveryWatcher, ...] + approved_observation_only: bool + + @classmethod + def parse(cls, value: object) -> RecoveryEnrollment: + raw = _object(value, cls.__dataclass_fields__, "recovery enrollment") + if raw["schema"] != "approved-management-recovery-containment-v1" or raw["profile"] != RECOVERY_PROFILE or raw["approved_observation_only"] is not True: + raise RecoveryContainmentUnavailable("recovery enrollment profile or authority differs") + values, controllers = raw["originals"], raw["watchers"] + if not isinstance(values, list) or not 1 <= len(values) <= _MAX_GENERATIONS or not isinstance(controllers, list) or not 1 <= len(controllers) <= 32: + raise RecoveryContainmentUnavailable("recovery original/controller set is absent or oversized") + originals = tuple(RecoveryOriginal.parse(item) for item in values) + target = RecoveryTarget.parse(raw["target"]) + watchers = tuple(RecoveryWatcher.parse(item) for item in controllers) + if 3 + 2 * sum(item.state == "started" for item in originals) + len(watchers) > 64: + raise RecoveryContainmentUnavailable("recovery scope exceeds the complete peer resource bound") + names = tuple(item.generation_id for item in originals) + units = tuple(item.unit for item in watchers) + if names != tuple(sorted(set(names))) or units != tuple(sorted(set(units))): + raise RecoveryContainmentUnavailable("recovery members or controllers are duplicate or unsorted") + ids = [target.management_id] + [identifier for item in originals for identifier in (item.management_id, item.proxy_id)] + instances = [target.instance_id] + [item.instance_id for item in originals] + if len(ids) != len(set(ids)) or len(instances) != len(set(instances)): + raise RecoveryContainmentUnavailable("recovery container or instance identities alias") + locks = [item.owner for item in originals if item.owner is not None] + [target.owner] + [item.lock for item in watchers] + if len({item.path for item in locks}) != len(locks) or len({(item.device, item.inode) for item in locks}) != len(locks): + raise RecoveryContainmentUnavailable("recovery owner/controller objects alias") + return cls( + str(raw["schema"]), + _text(raw["deployment_id"], _IDENTIFIER, "deployment"), + _text(raw["enrollment_id"], _IDENTIFIER, "enrollment"), + _text(raw["authorization_id"], _IDENTIFIER, "authorization"), + RECOVERY_PROFILE, + _text(raw["engine_id"], _IDENTIFIER, "engine"), + _text(raw["source_identity"], _IMAGE, "package source"), + originals, + target, + watchers, + True, + ) + + @property + def container_ids(self) -> tuple[str, ...]: + return tuple(sorted((self.target.management_id, *(identifier for item in self.originals for identifier in (item.management_id, item.proxy_id))))) + + +@dataclass(frozen=True) +class RecoveryEnrollmentObservation: + enrollment: RecoveryEnrollment + sha256: str + device: int + inode: int + ctime_ns: int + + +@dataclass(frozen=True) +class NamedDescriptor: + name: str + descriptor: int + + @classmethod + def parse(cls, value: object) -> NamedDescriptor: + raw = _object(value, cls.__dataclass_fields__, "recovery descriptor") + descriptor = raw["descriptor"] + if type(descriptor) is not int or not 0 <= descriptor <= 1048576: + raise RecoveryContainmentUnavailable("recovery descriptor differs") + return cls(_text(raw["name"], _IDENTIFIER, "resource name"), descriptor) + + +@dataclass(frozen=True) +class RecoveryResources: + holder_pid: int + holder_start_token: str + holder_boot_id: str + original_owner_fds: tuple[NamedDescriptor, ...] + controller_fds: tuple[NamedDescriptor, ...] + namespace_fds: tuple[NamedDescriptor, ...] + + @classmethod + def parse(cls, value: object) -> RecoveryResources: + raw = _object(value, cls.__dataclass_fields__, "recovery resources") + pid = _positive(raw["holder_pid"]) + if pid <= 1 or pid > 2147483647: + raise RecoveryContainmentUnavailable("recovery resource holder PID differs") + groups = [] + for name in ("original_owner_fds", "controller_fds", "namespace_fds"): + selected = raw[name] + if not isinstance(selected, list) or len(selected) > 32: + raise RecoveryContainmentUnavailable("recovery resource set is oversized") + group = tuple(NamedDescriptor.parse(item) for item in selected) + names = tuple(item.name for item in group) + if names != tuple(sorted(set(names))) or len({item.descriptor for item in group}) != len(group): + raise RecoveryContainmentUnavailable("recovery resource set is duplicate or unsorted") + groups.append(group) + if len({item.descriptor for group in groups for item in group}) != sum(map(len, groups)): + raise RecoveryContainmentUnavailable("recovery resource descriptor groups alias") + return cls(pid, _text(raw["holder_start_token"], re.compile(r"[1-9][0-9]{0,23}"), "holder lifetime"), _boot(raw["holder_boot_id"]), *groups) diff --git a/src/auto_coder/management/recovery_containment_profile.py b/src/auto_coder/management/recovery_containment_profile.py new file mode 100644 index 000000000..016330560 --- /dev/null +++ b/src/auto_coder/management/recovery_containment_profile.py @@ -0,0 +1,306 @@ +"""Actual source, configuration and bind profiles for ordinary recovery only.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +from dataclasses import dataclass +from pathlib import Path + +from .archived_exclusions import unique_pairs +from .containment_enrollment import _file_identity, _protected, _trusted_ancestors +from .containment_route_manifest import DeploymentRootBinding +from .installed_bootstrap_mount_profile import _mounts, _unshadowed_bootstrap_execution +from .original_attempts import source_identity +from .recovery_containment_model import RecoveryContainmentUnavailable, RecoveryEnrollment, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryTarget +from .recovery_source_profile import read_image_source_identity + +_COMMAND = ["-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json"] + + +@dataclass(frozen=True) +class RecoveryManagerFact: + generation_id: str + instance_id: str + container_id: str + image_id: str + state: str + source_identity: str + inspection_sha256: str + bootstrap_config_sha256: str + management_config_sha256: str + state_root: str + journal_root: str + oauth_root: str + owner: RecoveryLock | None + published_port: int + mounts: tuple[DeploymentRootBinding, ...] + + +def digest(value: object) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def read_bound_file(item: RecoveryFile, *, source: bool = False, expected_uid: int = 0) -> bytes: + path = Path(item.path) + if not source: + data, info = _protected(path, uid=expected_uid, size_limit=1024 * 1024) + else: + if path != path.resolve(strict=True): + raise RecoveryContainmentUnavailable("recovery controller source path is not canonical") + _trusted_ancestors(path.parent, uid=expected_uid) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_uid != expected_uid or info.st_nlink != 1 or info.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + raise RecoveryContainmentUnavailable("recovery controller source is not protected") + with os.fdopen(os.dup(fd), "rb") as stream: + data = stream.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024 or _file_identity(os.fstat(fd)) != _file_identity(info) or _file_identity(path.lstat()) != _file_identity(info): + raise RecoveryContainmentUnavailable("recovery source changed during readback") + finally: + os.close(fd) + if info.st_nlink != 1 or (info.st_dev, info.st_ino, hashlib.sha256(data).hexdigest()) != (item.device, item.inode, item.sha256): + raise RecoveryContainmentUnavailable("protected recovery source/configuration changed") + return data + + +def _security(inspected: dict[str, object]) -> tuple[dict[str, object], dict[str, object]]: + host, config = inspected.get("HostConfig"), inspected.get("Config") + if not isinstance(host, dict) or not isinstance(config, dict): + raise RecoveryContainmentUnavailable("recovery Docker command/security observation is absent") + if ( + host.get("Privileged") is not False + or host.get("PidMode") not in ("", None) + or host.get("CapAdd") not in (None, []) + or host.get("Devices") not in (None, []) + or host.get("SecurityOpt") not in (None, []) + or host.get("NetworkMode") in ("host", "none", None, "") + or config.get("User") != "" + or config.get("Healthcheck") is not None + ): + raise RecoveryContainmentUnavailable("recovery ordinary Docker security profile differs") + return host, config + + +def bind_mounts(container: dict[str, object], selected: tuple[DeploymentRootBinding, ...], *, expected_uid: int = 0) -> tuple[DeploymentRootBinding, ...]: + identifier = container.get("Id") + observed = _mounts(container) + expected = tuple(item for item in selected if item.container_id == identifier) + if tuple(sorted((item.source, item.destination, item.writable) for item in expected)) != observed: + raise RecoveryContainmentUnavailable("recovery actual and protected bind inventories differ") + for item in expected: + path = Path(item.source) + if path != path.resolve(strict=True) or path == Path("/") or path == Path("/proc") or path.is_relative_to("/proc") or path == Path("/sys") or path.is_relative_to("/sys") or path == Path("/dev") or path.is_relative_to("/dev"): + raise RecoveryContainmentUnavailable("recovery bind source is noncanonical or unsupported") + _trusted_ancestors(path.parent, uid=expected_uid) + info = path.lstat() + if info.st_uid != expected_uid or info.st_mode & (stat.S_IWGRP | stat.S_IWOTH) or (info.st_dev, info.st_ino, info.st_mode) != (item.device, item.inode, item.mode): + raise RecoveryContainmentUnavailable("recovery bind root was replaced or is unprotected") + return expected + + +def _mapped_file(path: str, mounts: tuple[DeploymentRootBinding, ...], *, writable: bool) -> str: + destination = Path(path) + matched = [item for item in mounts if destination == Path(item.destination) or Path(item.destination) in destination.parents] + if len(matched) != 1 or matched[0].writable != writable: + raise RecoveryContainmentUnavailable("recovery effective configuration is absent, shadowed or writable") + binding = matched[0] + return str(Path(binding.source) / destination.relative_to(binding.destination)) + + +def _root(path: object, mounts: tuple[DeploymentRootBinding, ...], *, writable: bool) -> str: + if not isinstance(path, str) or not Path(path).is_absolute() or ".." in Path(path).parts: + raise RecoveryContainmentUnavailable("recovery configured root differs") + found = [item for item in mounts if item.destination == path] + if len(found) != 1 or found[0].writable != writable or not stat.S_ISDIR(found[0].mode): + raise RecoveryContainmentUnavailable("recovery effective root mapping differs") + if any(item.destination != path and (Path(path) in Path(item.destination).parents or Path(item.destination) in Path(path).parents) for item in mounts): + raise RecoveryContainmentUnavailable("recovery effective root is shadowed") + return found[0].source + + +def _ordinary_mount_effects(config: dict[str, object], mounts: tuple[DeploymentRootBinding, ...], *, target: bool) -> None: + """Keep extra mounts within the explicit ordinary data/custody profile.""" + effects = {"/bootstrap": False, "/bootstrap/config.json": False, "/bootstrap/management.yml": False} + for name in ("state_root", "journal_root", "oauth_root"): + value = config.get(name) + if isinstance(value, str): + effects[value] = True + for name in ("management_version_root", "management_config", "original_genesis", "recovery_epoch_config"): + value = config.get(name) + if isinstance(value, str): + effects[value] = False + runtimes = config.get("runtimes", []) + exclusions = config.get("archived_exclusions", []) + if not isinstance(runtimes, list) or not isinstance(exclusions, list): + raise RecoveryContainmentUnavailable("recovery runtime/exclusion profile is ambiguous") + for runtime in runtimes: + if isinstance(runtime, dict) and isinstance(runtime.get("arguments"), list): + arguments = runtime["arguments"] + if "--config" in arguments and arguments.index("--config") + 1 < len(arguments) and isinstance(arguments[arguments.index("--config") + 1], str): + effects[arguments[arguments.index("--config") + 1]] = False + if target: + effects.update({"/management-recovery": False, "/management-recovery-history/history.lock": False}) + else: + if config.get("original_genesis") is not None: + effects["/management-original-history"] = False + if config.get("fresh_guard_path") == "/management-initialization/fresh-guard.sock": + effects["/management-initialization"] = False + if config.get("fresh_prior_lock_path") == "/management-prior-owner.lock": + effects.update({"/management-prior-owner.lock": False, "/management-prior-journal": False, "/management-prior-operations": False}) + for exclusion in exclusions: + if isinstance(exclusion, dict): + for name in ("journal_root", "fence"): + if isinstance(exclusion.get(name), str): + effects[exclusion[name]] = False + for item in mounts: + declared_owner = target and re.fullmatch(r"/management-recovery-originals/(?:0|[1-9][0-9]?)/owner\.lock", item.destination) is not None + worker_secret = re.fullmatch(r"/run/secrets/(?:beta|release)_worker_bridge", item.destination) is not None + if worker_secret and not item.writable and stat.S_ISREG(item.mode): + continue + if declared_owner: + if item.writable or not stat.S_ISREG(item.mode): + raise RecoveryContainmentUnavailable("recovery original custody mount is not a read-only file") + elif effects.get(item.destination) is not item.writable: + raise RecoveryContainmentUnavailable("recovery ordinary profile has an undeclared mount effect") + + +def validate_manager_profile(selected: RecoveryOriginal | RecoveryTarget, inspected: dict[str, object], *, source: str, expected_uid: int = 0) -> RecoveryManagerFact: + """Inspect current source/config shape without reading or promoting outcomes.""" + target = isinstance(selected, RecoveryTarget) + if inspected.get("Id") != selected.management_id or inspected.get("Image") != selected.management_image_id: + raise RecoveryContainmentUnavailable("recovery manager image or identity differs") + host, command = _security(inspected) + expected_commands = [_COMMAND + ["--recovery-epoch"]] if target else [_COMMAND, _COMMAND + ["--fresh-initialization"]] + if command.get("Entrypoint") != ["python"] or command.get("Cmd") not in expected_commands or command.get("WorkingDir") != "/workspace" or str(host.get("NetworkMode")).startswith("container:"): + raise RecoveryContainmentUnavailable("recovery ordinary bootstrap command differs") + mounts = bind_mounts(inspected, selected.mounts, expected_uid=expected_uid) + _unshadowed_bootstrap_execution(command, _mounts(inspected)) + environment = dict(item.split("=", 1) for item in cast_environment(command.get("Env"))) + supported_python = {"PYTHON_VERSION", "PYTHON_SHA256", "PYTHONUNBUFFERED", "PYTHONDONTWRITEBYTECODE"} + if environment.get("PYTHONDONTWRITEBYTECODE") != "1" or any(name.startswith("LD_") or name.startswith("PYTHON") and name not in supported_python for name in environment): + raise RecoveryContainmentUnavailable("recovery source execution environment is unsupported") + config_bytes = read_bound_file(selected.bootstrap_config, expected_uid=expected_uid) + from .bootstrap_supervisor import BootstrapConfig + + BootstrapConfig.read(Path(selected.bootstrap_config.path)) + read_bound_file(selected.management_config, expected_uid=expected_uid) + config = json.loads(config_bytes, object_pairs_hook=unique_pairs) + if ( + not isinstance(config, dict) + or config.get("instance_id") != selected.instance_id + or config.get("state_root") != "/management-child-state" + or config.get("oauth_root") != "/management-oauth" + or config.get("new_instance_initialization", False) is not False + or config.get("new_docker_network") is not None + or config.get("new_docker_host_port") is not None + ): + raise RecoveryContainmentUnavailable("recovery ordinary effective bootstrap identity differs") + _ordinary_mount_effects(config, mounts, target=target) + if _mapped_file("/bootstrap/config.json", mounts, writable=False) != selected.bootstrap_config.path: + raise RecoveryContainmentUnavailable("recovery bootstrap configuration is not actually mounted") + state_root = _root(config.get("state_root"), mounts, writable=True) + journal_root = _root(config.get("journal_root"), mounts, writable=True) + oauth_root = _root(config.get("oauth_root"), mounts, writable=True) + roots = (state_root, journal_root, oauth_root) + if len(set(roots)) != 3 or any(Path(left) == Path(right) or Path(left) in Path(right).parents or Path(right) in Path(left).parents for index, left in enumerate(roots) for right in roots[index + 1 :]): + raise RecoveryContainmentUnavailable("recovery mutable roots overlap") + if selected.owner is not None and selected.owner.path != str(Path(state_root) / ".supervisor-owner.lock"): + raise RecoveryContainmentUnavailable("recovery owner is not the effective supervisor owner file") + if isinstance(selected, RecoveryOriginal) and selected.state == "never-started" and (Path(state_root) / ".supervisor-owner.lock").exists(): + raise RecoveryContainmentUnavailable("never-started original already has an owner artifact") + runtimes = config.get("runtimes") + if not isinstance(runtimes, list) or not 1 <= len(runtimes) <= 64 or any(not isinstance(item, dict) for item in runtimes): + raise RecoveryContainmentUnavailable("recovery runtime configuration is absent or ambiguous") + configured = config.get("management_config") + version_root = config.get("management_version_root") + for runtime in runtimes: + if not isinstance(runtime, dict): + raise RecoveryContainmentUnavailable("recovery runtime configuration differs") + arguments = runtime.get("arguments") + if runtime.get("execution_kind") == "management-installed": + if not isinstance(configured, str) or not isinstance(version_root, str) or _root(version_root, mounts, writable=False) != version_root or arguments != ["-I", "-B", "-m", "auto_coder.cli", "management", "serve", "--config", configured]: + raise RecoveryContainmentUnavailable("recovery installed runtime mapping differs") + executable, source_root = runtime.get("executable"), runtime.get("source_root") + if not isinstance(executable, str) or not isinstance(source_root, str) or not Path(executable).is_relative_to(source_root) or not Path(source_root).is_relative_to(version_root) or source_root == version_root or any(".." in Path(item).parts for item in (executable, source_root)): + raise RecoveryContainmentUnavailable("recovery installed source path is unsupported") + management_path = configured + else: + if target or runtime.get("executable") != "/opt/auto-coder/.venv/bin/auto-coder" or not isinstance(arguments, list) or len(arguments) != 4 or arguments[:3] != ["management", "serve", "--config"] or not isinstance(arguments[3], str): + raise RecoveryContainmentUnavailable("recovery ordinary runtime command differs") + management_path = arguments[3] + if _mapped_file(management_path, mounts, writable=False) != selected.management_config.path: + raise RecoveryContainmentUnavailable("recovery runtime uses another management configuration") + if target: + if config.get("recovery_epoch_config") != "/bootstrap/recovery-epoch.json" or config.get("original_genesis") is not None or any(config.get(name) is not None for name in ("fresh_guard_path", "fresh_prior_lock_path", "fresh_runtime_binding")): + raise RecoveryContainmentUnavailable("recovery target lacks its separate prelaunch profile") + else: + if config.get("recovery_epoch_config") is not None: + raise RecoveryContainmentUnavailable("an original cannot be relabeled as the recovery target") + genesis = config.get("original_genesis") + if genesis is not None: + if genesis != "/bootstrap/original-genesis.json" or command.get("Cmd") != _COMMAND: + raise RecoveryContainmentUnavailable("ordinary original genesis configuration differs") + _mapped_file(genesis, mounts, writable=False) + _root("/management-original-history", mounts, writable=False) + elif command.get("Cmd") != _COMMAND + ["--fresh-initialization"]: + raise RecoveryContainmentUnavailable("ordinary original lacks genesis/history-capable entrypoint") + binding = host.get("PortBindings") + if not isinstance(binding, dict) or set(binding) != {"8080/tcp"} or not isinstance(binding["8080/tcp"], list) or len(binding["8080/tcp"]) != 1: + raise RecoveryContainmentUnavailable("recovery published endpoint is ambiguous") + endpoint = binding["8080/tcp"][0] + if not isinstance(endpoint, dict) or set(endpoint) != {"HostIp", "HostPort"} or endpoint["HostIp"] != "127.0.0.1" or not isinstance(endpoint["HostPort"], str) or not endpoint["HostPort"].isdecimal() or not 0 < int(endpoint["HostPort"]) <= 65535: + raise RecoveryContainmentUnavailable("recovery published endpoint is not exact loopback TCP") + port = int(endpoint["HostPort"]) + if isinstance(selected, RecoveryTarget) and port != selected.published_port: + raise RecoveryContainmentUnavailable("recovery target published endpoint differs") + if source != source_identity() or read_image_source_identity(selected.management_id) != source: + raise RecoveryContainmentUnavailable("recovery actual complete package source differs") + generation = "target" if isinstance(selected, RecoveryTarget) else selected.generation_id + state = "prelaunch-target" if isinstance(selected, RecoveryTarget) else selected.state + return RecoveryManagerFact(generation, selected.instance_id, selected.management_id, selected.management_image_id, state, source, digest([inspected]), selected.bootstrap_config.sha256, selected.management_config.sha256, state_root, journal_root, oauth_root, selected.owner, port, mounts) + + +def cast_environment(value: object) -> tuple[str, ...]: + if not isinstance(value, list) or any(not isinstance(item, str) or "=" not in item for item in value): + raise RecoveryContainmentUnavailable("recovery process environment is ambiguous") + return tuple(value) + + +def validate_proxy_profile(selected: RecoveryOriginal, inspected: dict[str, object], *, expected_uid: int = 0) -> str: + if inspected.get("Id") != selected.proxy_id or inspected.get("Image") != selected.proxy_image_id: + raise RecoveryContainmentUnavailable("original recovery proxy identity differs") + host, config = _security(inspected) + if host.get("NetworkMode") != "container:" + selected.management_id or config.get("Entrypoint") != ["caddy"] or config.get("Cmd") != ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] or config.get("WorkingDir") != "/srv": + raise RecoveryContainmentUnavailable("original recovery proxy command or network differs") + mounts = bind_mounts(inspected, selected.mounts, expected_uid=expected_uid) + if _mapped_file("/etc/caddy/Caddyfile", mounts, writable=False) != selected.proxy_config.path: + raise RecoveryContainmentUnavailable("original recovery proxy configuration is not actually mounted") + for item in mounts: + destination = Path(item.destination) + if any(destination == Path(root) or destination in Path(root).parents or Path(root) in destination.parents for root in ("/usr/bin", "/usr/local/bin", "/bin", "/lib", "/usr/lib", "/srv")): + raise RecoveryContainmentUnavailable("original recovery Caddy executable is shadowed") + read_bound_file(selected.proxy_config, expected_uid=expected_uid) + return digest([inspected]) + + +def assert_disjoint_target(enrollment: RecoveryEnrollment, facts: tuple[RecoveryManagerFact, ...]) -> None: + target = facts[-1] + if target.container_id != enrollment.target.management_id: + raise RecoveryContainmentUnavailable("recovery target fact is not exact") + for old in facts[:-1]: + if target.published_port == old.published_port: + raise RecoveryContainmentUnavailable("recovery target cannot hide an old published listener") + if target.oauth_root != old.oauth_root: + raise RecoveryContainmentUnavailable("recovery target does not retain the original OAuth root") + for left in (target.state_root, target.journal_root): + for right in (old.state_root, old.journal_root, old.oauth_root): + if Path(left) == Path(right) or Path(left) in Path(right).parents or Path(right) in Path(left).parents: + raise RecoveryContainmentUnavailable("recovery target mutable root overlaps an original") + old_root_ids = {(item.device, item.inode) for original in enrollment.originals for item in original.mounts if item.writable and item.source != target.oauth_root} + if any((item.device, item.inode) in old_root_ids for item in enrollment.target.mounts if item.writable): + raise RecoveryContainmentUnavailable("recovery target aliases an original mutable root") diff --git a/src/auto_coder/management/recovery_containment_resources.py b/src/auto_coder/management/recovery_containment_resources.py new file mode 100644 index 000000000..73f22da6f --- /dev/null +++ b/src/auto_coder/management/recovery_containment_resources.py @@ -0,0 +1,136 @@ +"""Read existing recovery fences and retained namespaces without acquiring them.""" + +from __future__ import annotations + +import os +import select +import stat +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +from .containment_enrollment import _trusted_ancestors +from .containment_kernel_subobservation import _external_namespace_pin, _holder_start_token +from .fresh_host_observation import _assert_pid_flock, assert_prior_network_pin +from .recovery_containment_model import RecoveryContainmentUnavailable, RecoveryEnrollment, RecoveryLock, RecoveryResources +from .recovery_epoch_peer import KernelLock, inspect_kernel_lock + + +@dataclass(frozen=True) +class RecoveryResourceFact: + name: str + kind: str + holder_descriptor: int + path: str + device: int + inode: int + mode: int + uid: int + kernel_lock: KernelLock | None + + +@dataclass(frozen=True) +class RecoveryResourceObservation: + holder_pid: int + holder_start_token: str + holder_boot_id: str + resources: tuple[RecoveryResourceFact, ...] + + +def _private_lock(path: Path, selected: RecoveryLock, *, uid: int) -> os.stat_result: + if path != path.resolve(strict=True) or str(path) != selected.path: + raise RecoveryContainmentUnavailable("recovery fence path is not canonical") + _trusted_ancestors(path.parent, uid=uid) + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != uid or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or (info.st_dev, info.st_ino) != (selected.device, selected.inode): + raise RecoveryContainmentUnavailable("original recovery fence is missing, replaced or unprotected") + return info + + +@contextmanager +def hold_observed_resources(enrollment: RecoveryEnrollment, resources: RecoveryResources, *, expected_uid: int = 0) -> Iterator[RecoveryResourceObservation]: + """Retain observation references and recheck every supplied actual holder. + + This takes no flock and creates no file. Original SH attribution must name + the issuer process itself, so an inherited shared OFD is not independent + issuer acquisition. Controller EX must name that same current issuer. + """ + originals = {item.generation_id: item for item in enrollment.originals if item.state == "started"} + watchers = {item.unit: item for item in enrollment.watchers} + owner_fds = {item.name: item.descriptor for item in resources.original_owner_fds} + controller_fds = {item.name: item.descriptor for item in resources.controller_fds} + namespace_fds = {item.name: item.descriptor for item in resources.namespace_fds} + if set(owner_fds) != set(originals) or set(namespace_fds) != set(originals) or set(controller_fds) != set(watchers): + raise RecoveryContainmentUnavailable("recovery resource membership differs from the protected scope") + with ExitStack() as stack: + lifetime = os.pidfd_open(resources.holder_pid, 0) + stack.callback(os.close, lifetime) + poller = select.poll() + poller.register(lifetime, select.POLLIN) + held: list[tuple[RecoveryResourceFact, int, RecoveryLock | None]] = [] + + def check_holder() -> None: + boot = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() + if boot != resources.holder_boot_id or _holder_start_token(resources.holder_pid) != resources.holder_start_token or poller.poll(0): + raise RecoveryContainmentUnavailable("recovery resource holder changed or exited") + proc = Path("/proc") / str(resources.holder_pid) + if proc.stat().st_uid != expected_uid or any(os.readlink(proc / "ns" / name) != os.readlink("/proc/1/ns/" + name) for name in ("pid", "user", "mnt", "net")): + raise RecoveryContainmentUnavailable("recovery resource holder is not the trusted host process") + + def check_one(fact: RecoveryResourceFact, descriptor: int, selected: RecoveryLock | None) -> None: + target = Path("/proc") / str(resources.holder_pid) / "fd" / str(fact.holder_descriptor) + info, remote = os.fstat(descriptor), target.stat() + expected = (fact.device, fact.inode, fact.mode, fact.uid) + if (info.st_dev, info.st_ino, info.st_mode, info.st_uid) != expected or (remote.st_dev, remote.st_ino, remote.st_mode, remote.st_uid) != expected or os.readlink(target) != fact.path: + raise RecoveryContainmentUnavailable("held recovery resource changed") + if selected is None: + assert_prior_network_pin(descriptor, fact.path) + return + _private_lock(Path(fact.path), selected, uid=expected_uid) + observed = inspect_kernel_lock(resources.holder_pid, fact.holder_descriptor, fact.device, fact.inode) + if observed != fact.kernel_lock or observed is None or observed.pid != resources.holder_pid: + raise RecoveryContainmentUnavailable("recovery fence was inherited, released or changed") + _assert_pid_flock(resources.holder_pid, descriptor, observed.mode) + + check_holder() + for kind, selected_fds in (("original-sh", owner_fds), ("controller-ex", controller_fds)): + for name, remote_fd in sorted(selected_fds.items()): + selected = originals[name].owner if kind == "original-sh" else watchers[name].lock + if selected is None: + raise RecoveryContainmentUnavailable("original owner identity is absent") + expected = _private_lock(Path(selected.path), selected, uid=expected_uid) + remote_path = Path("/proc") / str(resources.holder_pid) / "fd" / str(remote_fd) + descriptor = os.open(remote_path, os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK) + stack.callback(os.close, descriptor) + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_nlink) != (expected.st_dev, expected.st_ino, expected.st_mode, expected.st_uid, expected.st_nlink): + raise RecoveryContainmentUnavailable("recovery issuer fence names another object") + lock = inspect_kernel_lock(resources.holder_pid, remote_fd, info.st_dev, info.st_ino) + mode = "READ" if kind == "original-sh" else "WRITE" + if lock is None or lock.mode != mode or lock.pid != resources.holder_pid: + raise RecoveryContainmentUnavailable("recovery issuer lacks independently acquired " + kind) + fact = RecoveryResourceFact(name, kind, remote_fd, selected.path, info.st_dev, info.st_ino, info.st_mode, info.st_uid, lock) + check_one(fact, descriptor, selected) + held.append((fact, descriptor, selected)) + for name, remote_fd in sorted(namespace_fds.items()): + original = originals[name] + pin = original.namespace_pin + if pin is None or original.boot_id != resources.holder_boot_id: + raise RecoveryContainmentUnavailable("original namespace boot or pin is absent") + stack.enter_context(_external_namespace_pin(pin, pin.namespace)) + target = Path("/proc") / str(resources.holder_pid) / "fd" / str(remote_fd) + descriptor = os.open(target, os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK) + stack.callback(os.close, descriptor) + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino) != (pin.device, pin.inode): + raise RecoveryContainmentUnavailable("recovery namespace descriptor is not the retained original object") + fact = RecoveryResourceFact(name, "network-namespace", remote_fd, pin.namespace, info.st_dev, info.st_ino, info.st_mode, info.st_uid, None) + check_one(fact, descriptor, None) + held.append((fact, descriptor, None)) + check_holder() + result = RecoveryResourceObservation(resources.holder_pid, resources.holder_start_token, resources.holder_boot_id, tuple(item[0] for item in held)) + yield result + for fact, descriptor, selected in held: + check_one(fact, descriptor, selected) + check_holder() diff --git a/src/auto_coder/management/recovery_epoch.py b/src/auto_coder/management/recovery_epoch.py new file mode 100644 index 000000000..5a3543edd --- /dev/null +++ b/src/auto_coder/management/recovery_epoch.py @@ -0,0 +1,377 @@ +"""Deployment-fixed one-shot host issuer for management recovery ownership. + +Original evidence and current containment are independently re-derived here. +Only this issuer combines those facts with exact target READY and physically +verified consumer custody, durably consumes the epoch, and sends one grant. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import signal +import socket +import stat +import sys +import threading +from contextlib import ExitStack, contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterator + +from auto_coder.logger_config import get_logger, setup_logger + +from .bootstrap_supervisor import ManagementBootstrapSupervisor +from .containment_kernel_subobservation import _require_capture_host +from .installed_bootstrap_mount_profile import _inspect as inspect_container +from .original_attempt_evidence import OriginalLineageEvidence, derive_lineage, verify_lineage +from .original_attempts import canonical, source_identity +from .quarantine_history import InspectionReport, QuarantineHistory, QuarantineJournal +from .recovery_containment import RecoveryContainmentReport, load_enrollment +from .recovery_containment import observe as observe_containment +from .recovery_containment import verify as verify_containment +from .recovery_containment_model import RecoveryEnrollment, RecoveryEnrollmentObservation +from .recovery_epoch_authorization import ( + PEER_ROOT, + PEER_SOCKET, + AuthorizationObservation, + EpochAuthorization, + TargetReadyObservation, + check_authorization, + load_authorization, + observe_target_ready, + read_bound_file, + verify_target_ready, +) +from .recovery_epoch_consumer import GRANT_SCHEMA, parse_consumer_configuration +from .recovery_epoch_peer import ExpectedPeer, HeldFence, HeldResource, PeerCustody, PeerSession, encode_peer_message, inspect_held_resource +from .recovery_epoch_resources import EpochResources, hold_epoch_resources +from .recovery_epoch_state import EpochBinding, EpochInventory, EpochReceipt, EpochUnavailable, RecoveryEpochState, hold_recovery_epoch, issuer_lifetime + +logger = get_logger(__name__) +_OVERRIDES = ("AUTO_CODER_RECOVERY_EPOCH_ROOT", "AUTO_CODER_RECOVERY_EPOCH_INVENTORY", "AUTO_CODER_RECOVERY_AUTHORIZATION_ROOT", "AUTO_CODER_RECOVERY_CONTAINMENT_ROOT") + + +@dataclass(frozen=True) +class VerifiedEpochEvidence: + lineage: OriginalLineageEvidence + containment: RecoveryContainmentReport + target: TargetReadyObservation + inspection: InspectionReport + custody: PeerCustody + + +@dataclass(frozen=True) +class EpochDispatch: + receipt: EpochReceipt + evidence_sha256: str + dispatch_attempted: bool + # A successful send is not acknowledgment, child launch or current readiness. + current_ready: bool = False + + +def join_lineage(authorization: EpochAuthorization, inventory: EpochInventory, enrollment: RecoveryEnrollmentObservation, lineage: OriginalLineageEvidence, containment: RecoveryContainmentReport) -> None: + """Match actual complete original derivation to the entire observed scope.""" + selected = enrollment.enrollment + if lineage.unavailable or lineage.genesis is None or not lineage.generations or lineage.binding != authorization.history: + raise EpochUnavailable("complete original genesis and retained lineage are unavailable") + old_attempts = {generation.request_id for generation in lineage.generations} + if authorization.request_id in old_attempts or authorization.authorization_id == lineage.genesis.authorization_id: + raise EpochUnavailable("recovery requires new authorization and a new request identity") + if containment.status != "observed-current-recovery-containment" or containment.grant_issued or containment.history_asserted or containment.admission_created or containment.enrollment != enrollment: + raise EpochUnavailable("current containment is not the exact non-authorizing prerequisite") + if selected.deployment_id != lineage.binding.deployment or selected.source_identity != lineage.binding.source or selected.source_identity != source_identity() or selected.source_identity != "sha256:" + inventory.source_sha256: + raise EpochUnavailable("original and current source/deployment identities differ") + if selected.target.management_id != inventory.target_id or selected.target.bootstrap_config.sha256 != inventory.config_sha256: + raise EpochUnavailable("exact target differs from deployment-fixed inventory") + if selected.engine_id != lineage.genesis.engine_id: + raise EpochUnavailable("current Docker engine differs from original genesis") + original_by_id = {item.management_id: item for item in selected.originals} + generations = {item.management_id: item for item in lineage.generations} + facts = {item.container_id: item for item in containment.managers} + if len(generations) != len(lineage.generations) or len(facts) != len(containment.managers) or set(original_by_id) - set(generations): + raise EpochUnavailable("current recovery enrollment omits or aliases original lineage") + if set(facts) != {*original_by_id, selected.target.management_id}: + raise EpochUnavailable("current effective manager scope differs from the exact enrollment") + for identifier, generation in generations.items(): + original = original_by_id.get(identifier) + if original is None: + if generation.outcome != "never-created" or identifier in containment.docker_ids or generation.owner_inode: + raise EpochUnavailable("original generation is absent from current full-scope containment") + continue + fact = facts[identifier] + if (original.instance_id, original.management_image_id, original.bootstrap_config.sha256, original.management_config.sha256) != (generation.instance_id, generation.image_id, generation.bootstrap_config_sha256, generation.management_config_sha256): + raise EpochUnavailable("current original configuration differs from authoritative lineage") + if generation.proxy_id and (original.proxy_id, original.proxy_image_id) != (generation.proxy_id, generation.proxy_image_id): + raise EpochUnavailable("original proxy differs from genesis provenance") + if (fact.state_root, fact.journal_root, fact.oauth_root) != (generation.state_root, generation.journal_root, generation.oauth_root): + raise EpochUnavailable("original root mapping differs from current effective configuration") + for path, device, inode in ((generation.state_root, generation.state_device, generation.state_inode), (generation.journal_root, generation.journal_device, generation.journal_inode), (generation.oauth_root, generation.oauth_device, generation.oauth_inode)): + roots = [mount for mount in fact.mounts if mount.source == path and mount.writable] + if len(roots) != 1 or (roots[0].device, roots[0].inode) != (device, inode): + raise EpochUnavailable("original root inode differs from current Docker mapping") + if generation.outcome == "never-created": + if original.state != "never-started" or original.owner is not None or generation.owner_inode: + raise EpochUnavailable("original never-created boundary conflicts with current facts") + elif original.state != "started" or original.owner is None or (original.owner.path, original.owner.device, original.owner.inode) != (generation.owner_path, generation.owner_device, generation.owner_inode): + raise EpochUnavailable("current original owner differs from native original provenance") + target = facts[selected.target.management_id] + oauth = lineage.genesis.root("oauth") + roots = [item for item in target.mounts if item.writable and item.source == oauth.host_path] + if target.oauth_root != oauth.host_path or len(roots) != 1 or (roots[0].device, roots[0].inode) != (oauth.device, oauth.inode): + raise EpochUnavailable("recovery target does not preserve the exact original OAuth store") + original_paths = [Path(value) for generation in lineage.generations for value in (generation.state_root, generation.journal_root, generation.oauth_root)] + for target_root in (Path(target.state_root), Path(target.journal_root)): + if any(target_root == original or target_root in original.parents or original in target_root.parents for original in original_paths): + raise EpochUnavailable("recovery target reuses or overlaps an original root") + + +def _quarantine_entries(lineage: OriginalLineageEvidence) -> tuple[QuarantineJournal, ...]: + originals = {item.path: item for item in lineage.originals} + result = [] + for generation in lineage.generations: + if generation.outcome == "never-created": + continue + stores = dict(generation.expected_stores) + original = originals.get(stores.get("operations", "")) + owner = originals.get(generation.owner_path) + if original is None or owner is None: + raise EpochUnavailable("original journal or owner fingerprint is missing") + database, fence = Path(original.path), Path(owner.path) + database_info, fence_info = database.lstat(), fence.lstat() + result.append(QuarantineJournal(generation.instance_id, database, fence, original.device, original.inode, original.sha256, database_info.st_ctime_ns, owner.device, owner.inode, fence_info.st_ctime_ns)) + return tuple(result) + + +def _check_consumer_scope(authorization: EpochAuthorization, enrollment: RecoveryEnrollment, resources: EpochResources) -> None: + selected = parse_consumer_configuration(read_bound_file(authorization.consumer_config)) + if (selected.request_id, selected.authorization_id, selected.target_id, selected.issuer_uid, selected.socket_directory_device, selected.socket_directory_inode) != ( + authorization.request_id, + authorization.authorization_id, + authorization.target_id, + 0, + authorization.socket_directory_device, + authorization.socket_directory_inode, + ): + raise EpochUnavailable("consumer's pinned authorization differs") + peer_mounts = [item for item in enrollment.target.mounts if item.destination == "/management-recovery"] + if len(peer_mounts) != 1 or peer_mounts[0].source != str(PEER_ROOT) or peer_mounts[0].writable or (peer_mounts[0].device, peer_mounts[0].inode) != (authorization.socket_directory_device, authorization.socket_directory_inode): + raise EpochUnavailable("consumer lacks the exact read-only issuer endpoint mount") + original_paths = {"history": ("/management-recovery-history/history.lock", str(Path(authorization.history.root) / "history.lock"))} + original_paths.update({f"owner-{index}": (f"/management-recovery-originals/{index}/owner.lock", item.owner.path) for index, item in enumerate(enrollment.originals) if item.owner is not None}) + held: tuple[HeldFence | HeldResource, ...] = (*resources.fences, *resources.namespaces) + observed = tuple(inspect_held_resource(item) for item in held) + originals = {(item.name, item.device, item.inode) for item in observed if item.kind == "original-sh"} + if {(item.name, item.device, item.inode) for item in selected.original_fences} != originals or len(selected.original_fences) != len(originals): + raise EpochUnavailable("consumer omitted or changed an independently acquired original fence") + for original in selected.original_fences: + container_path, host_path = original_paths[original.name] + mounts = [item for item in enrollment.target.mounts if item.destination == container_path] + if original.path != container_path or len(mounts) != 1 or mounts[0].source != host_path or mounts[0].writable or (mounts[0].device, mounts[0].inode) != (original.device, original.inode): + raise EpochUnavailable("consumer original fence is not the exact read-only mounted inode") + target = next(item for item in observed if item.kind == "target-ex") + if (selected.target_owner.name, selected.target_owner.device, selected.target_owner.inode) != (target.name, target.device, target.inode): + raise EpochUnavailable("consumer's independent target owner differs") + transfers = {(item.name, item.kind, item.device, item.inode) for item in observed if item.kind in ("transferred-ex", "network-namespace")} + if {(item.name, item.kind, item.device, item.inode) for item in selected.transferred_resources} != transfers or len(selected.transferred_resources) != len(transfers): + raise EpochUnavailable("consumer omitted or changed controller, deployment or namespace custody") + + +def _expected_peer(authorization: EpochAuthorization) -> ExpectedPeer: + actual = inspect_container(authorization.target_id) + state = actual.get("State") + if actual.get("Id") != authorization.target_id or not isinstance(state, dict) or state.get("Running") is not True or type(state.get("Pid")) is not int or state["Pid"] <= 1: + raise EpochUnavailable("exact recovery target is not a live bootstrap consumer") + pid = state["Pid"] + token = ManagementBootstrapSupervisor._start_token(pid) + if not token: + raise EpochUnavailable("exact recovery target process lifetime is unavailable") + return ExpectedPeer(pid, token, issuer_lifetime()[0], 0, authorization.request_id, authorization.authorization_id, authorization.target_id) + + +def _match_peer_report(session: PeerSession, report: RecoveryContainmentReport) -> None: + if (report.target_pid, report.target_start_token, report.facts.target.boot_id) != (session.expected.pid, session.expected.start_token, session.expected.boot_id): + raise EpochUnavailable("authenticated peer differs from currently observed Docker target") + + +def _binding(state: RecoveryEpochState, authorization: AuthorizationObservation, evidence: VerifiedEpochEvidence) -> EpochBinding: + boot, pid, start = issuer_lifetime() + # Retain all original file fingerprints and current scope/custody identities; + # the independently verified controller/kernel report is bound in full by + # its canonical evidence digest, not by an unverified success flag. + original = { + "schema": "verified-management-recovery-evidence-v1", + "authorization": asdict(authorization), + "lineage": asdict(evidence.lineage), + "containment_sha256": evidence.containment.evidence_sha256, + "containment_enrollment": asdict(evidence.containment.enrollment), + "controller_fences": [asdict(item) for item in evidence.containment.facts.watchers], + "current_holder": asdict(evidence.containment.facts.resources), + "target": asdict(evidence.target), + "inspection": asdict(evidence.inspection), + "custody": asdict(evidence.custody), + } + encoded = canonical(original) + inventory = state.inventory + return EpochBinding( + inventory.deployment_id, + inventory.authorization_id, + inventory.enrollment_ids, + inventory.request_id, + inventory.preparation_id, + inventory.target_id, + inventory.source_sha256, + inventory.config_sha256, + hashlib.sha256(encoded).hexdigest(), + boot, + pid, + start, + inventory.authorization_sha256, + encoded.decode(), + ) + + +def issue_epoch_once(state: RecoveryEpochState, authorization: AuthorizationObservation, enrollment: RecoveryEnrollmentObservation, resources: EpochResources, session: PeerSession) -> tuple[EpochDispatch, PeerCustody]: + """Verify, irreversibly consume and attempt exactly one current peer grant.""" + selected = enrollment.enrollment + approved = authorization.authorization + check_authorization(authorization, state.inventory) + _check_consumer_scope(approved, selected, resources) + target = observe_target_ready(approved, selected) + lineage = derive_lineage(approved.history, resources.history_descriptor, resources.original_descriptors) + lineage = verify_lineage(lineage, approved.history, resources.history_descriptor, resources.original_descriptors) + with QuarantineHistory(_quarantine_entries(lineage)) as history: + inspection = history.inspect() + containment = observe_containment(approved.enrollment_id, resources.containment) + join_lineage(approved, state.inventory, enrollment, lineage, containment) + _match_peer_report(session, containment) + custody = session.offer_and_verify_custody(resources.fences, resources.namespaces) + evidence = VerifiedEpochEvidence(lineage, containment, target, inspection, custody) + binding = _binding(state, authorization, evidence) + state.reserve(binding) + state.record_intent(binding) + + def final_verification() -> None: + check_authorization(authorization, state.inventory) + if load_enrollment(approved.enrollment_id) != enrollment: + raise EpochUnavailable("protected recovery enrollment changed at issuance") + verify_target_ready(approved, selected, target) + verify_lineage(lineage, approved.history, resources.history_descriptor, resources.original_descriptors) + if history.inspect() != inspection: + raise EpochUnavailable("quarantined original exclusions changed before grant") + current = verify_containment(approved.enrollment_id, containment, resources.containment) + join_lineage(approved, state.inventory, enrollment, lineage, current) + _match_peer_report(session, current) + _check_consumer_scope(approved, selected, resources) + resources.check() + session.check(custody) + state.check() + + final_verification() + receipt = state.consume(binding) + # Persistence can block or be interrupted. Its completion is never + # substituted for a current containment/custody/READY observation. + final_verification() + packet = encode_peer_message( + { + "schema": GRANT_SCHEMA, + "granted": "recovery-epoch-consumed", + **{name: getattr(approved, name) for name in ("request_id", "authorization_id", "target_id")}, + "challenge": custody.challenge, + "receipt_nonce": custody.receipt_nonce, + **{name: getattr(receipt, name) for name in ("inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256")}, + "host_boot_id": binding.host_boot_id, + "host_pid": binding.host_pid, + "host_start_token": binding.host_start_token, + } + ) + session.check(custody) + state.check() + # A partial/failed send, lost reply or exception has no retry path. + session.connection.settimeout(10) + session.connection.sendall(packet) + logger.bind(recovery_epoch_phase="grant-send-completed", request_id=approved.request_id, target_id=approved.target_id, current_ready=False).info("One consumed recovery epoch grant send completed; delivery and readiness are not inferred") + return EpochDispatch(receipt, binding.evidence_sha256, True), custody + + +@contextmanager +def _listen(authorization: AuthorizationObservation) -> Iterator[socket.socket]: + root = PEER_ROOT.stat() + expected = authorization.authorization + if (root.st_dev, root.st_ino) != (expected.socket_directory_device, expected.socket_directory_inode) or PEER_SOCKET.exists() or PEER_SOCKET.is_symlink(): + raise EpochUnavailable("fixed recovery socket directory or original endpoint is unavailable") + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + bound = None + try: + listener.bind(str(PEER_SOCKET)) + os.chmod(PEER_SOCKET, 0o600) + bound = PEER_SOCKET.lstat() + if not stat.S_ISSOCK(bound.st_mode) or bound.st_uid != 0: + raise EpochUnavailable("recovery local endpoint is not protected") + listener.listen(1) + listener.settimeout(600) + logger.bind(recovery_epoch_phase="listening", request_id=expected.request_id, target_id=expected.target_id).info("Fixed recovery issuer endpoint is listening for its exact target") + yield listener + finally: + listener.close() + if bound is not None: + try: + current = PEER_SOCKET.lstat() + if (current.st_dev, current.st_ino) == (bound.st_dev, bound.st_ino): + PEER_SOCKET.unlink() + except FileNotFoundError: + pass + + +def run_issuer() -> EpochDispatch: + """Own one current epoch until the target exits or its custody is lost.""" + if any(name in os.environ for name in _OVERRIDES): + raise EpochUnavailable("request/environment recovery inventory overrides are unsupported") + _require_capture_host() + stopped = threading.Event() + handlers: dict[int, object] = {} + with ExitStack() as held: + state = held.enter_context(hold_recovery_epoch()) + authorization = load_authorization(state.inventory) + enrollment = load_enrollment(authorization.authorization.enrollment_id) + resources = held.enter_context(hold_epoch_resources(authorization.authorization, enrollment.enrollment, state)) + listener = held.enter_context(_listen(authorization)) + connection, _ = listener.accept() + try: + session = held.enter_context(PeerSession(connection, _expected_peer(authorization.authorization), timeout=60)) + except BaseException: + connection.close() + raise + listener.close() + dispatch, custody = issue_epoch_once(state, authorization, enrollment, resources, session) + for signum in (signal.SIGTERM, signal.SIGINT): + handlers[signum] = signal.signal(signum, lambda *_: stopped.set()) + try: + while not stopped.wait(0.2): + check_authorization(authorization, state.inventory) + resources.check() + session.check(custody) + state.check() + return dispatch + finally: + for saved_signal, handler in handlers.items(): + signal.signal(saved_signal, handler) # type: ignore[arg-type] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Issue one approved management ownership epoch from fixed host authority") + parser.parse_args(argv) + setup_logger(stream=sys.stdout, log_file="/var/log/auto-coder/recovery-epoch.log") + try: + result = run_issuer() + except (OSError, ValueError, TypeError, KeyError, RuntimeError) as error: + logger.bind(recovery_epoch_phase="unavailable", current_ready=False).error("Recovery epoch unavailable; preserved evidence is not repaired: {}", error) + return 2 + print(json.dumps(asdict(result), sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/auto_coder/management/recovery_epoch_authorization.py b/src/auto_coder/management/recovery_epoch_authorization.py new file mode 100644 index 000000000..75e796a6b --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_authorization.py @@ -0,0 +1,236 @@ +"""Protected fixed recovery authorization and independently checked READY. + +An authorization selects a finite scope. It neither establishes original +history nor replaces the issuer's current Docker, kernel and custody checks. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +from auto_coder.operating_policy import MANAGEMENT_REPOSITORY + +from .archived_exclusions import unique_pairs +from .bootstrap_supervisor import BootstrapConfig +from .bootstrap_upgrade import _source_authority +from .containment_enrollment import _approved_directory +from .original_attempts import HistoryBinding, canonical, read_original +from .recovery_containment_model import RecoveryEnrollment, RecoveryFile +from .recovery_epoch_state import EpochInventory, EpochUnavailable +from .self_update_handoff import runtime_for_receipt +from .version_staging import ManagementReadyReceipt, ManagementRuntimeConfig, ManagementVersionStager + +AUTHORIZATION_ROOT = Path("/etc/auto-coder/recovery-epoch") +AUTHORIZATION_FILE = AUTHORIZATION_ROOT / "authorization.json" +PEER_ROOT = Path("/var/lib/auto-coder/recovery-epoch-peer") +PEER_SOCKET = PEER_ROOT / "issuer.sock" +AUTHORIZATION_SCHEMA = "management-recovery-authorization-v1" +_SHA = re.compile(r"[0-9a-f]{40}\Z", re.ASCII) +_TOKEN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z", re.ASCII) + + +@dataclass(frozen=True) +class EpochAuthorization: + schema: str + deployment_id: str + authorization_id: str + request_id: str + preparation_id: str + target_id: str + enrollment_id: str + history: HistoryBinding + consumer_config: RecoveryFile + ready_receipt: RecoveryFile + source_repository: str + source_token_file: str + python_executable: str + uv_executable: str + uv_version: str + source_sha: str + socket_directory_device: int + socket_directory_inode: int + + @classmethod + def parse(cls, value: object) -> EpochAuthorization: + if not isinstance(value, dict) or set(value) != set(cls.__dataclass_fields__): + raise EpochUnavailable("complete fixed recovery authorization required") + raw = dict(value) + history = raw["history"] + if not isinstance(history, dict) or set(history) != set(HistoryBinding.__dataclass_fields__) or any(not isinstance(item, str) for item in history.values()): + raise EpochUnavailable("original history authorization is incomplete") + raw["history"] = HistoryBinding(**history) + raw["consumer_config"] = RecoveryFile.parse(raw["consumer_config"]) + raw["ready_receipt"] = RecoveryFile.parse(raw["ready_receipt"]) + result = cls(**raw) + if result.schema != AUTHORIZATION_SCHEMA: + raise EpochUnavailable("unsupported recovery authorization schema") + for field in ("deployment_id", "authorization_id", "request_id", "preparation_id", "enrollment_id"): + token = getattr(result, field) + if not isinstance(token, str) or not _TOKEN.fullmatch(token): + raise EpochUnavailable("invalid recovery authorization " + field) + if not isinstance(result.target_id, str) or not re.fullmatch(r"[0-9a-f]{64}", result.target_id) or not isinstance(result.source_sha, str) or not _SHA.fullmatch(result.source_sha): + raise EpochUnavailable("exact recovery target and source required") + if result.history.deployment != result.deployment_id: + raise EpochUnavailable("history names another recovery deployment") + for path in (result.history.root, result.source_repository, result.source_token_file, result.python_executable, result.uv_executable): + _path(path) + if not isinstance(result.uv_version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", result.uv_version): + raise EpochUnavailable("exact configured installer version required") + for field in ("socket_directory_device", "socket_directory_inode"): + if type(getattr(result, field)) is not int or getattr(result, field) <= 0: + raise EpochUnavailable("fixed recovery peer directory identity required") + return result + + +@dataclass(frozen=True) +class AuthorizationObservation: + authorization: EpochAuthorization + original: RecoveryFile + + +@dataclass(frozen=True) +class TargetReadyObservation: + receipt: ManagementReadyReceipt + bootstrap_config: RecoveryFile + management_config: RecoveryFile + consumer_config: RecoveryFile + ready_receipt: RecoveryFile + runtime_sha256: str + + +def _path(value: object) -> Path: + if not isinstance(value, str) or not value or len(value) > 4096: + raise EpochUnavailable("invalid recovery authority path") + path = Path(value) + if not path.is_absolute() or str(path) != value or path == Path("/") or ".." in path.parts or "\x00" in value: + raise EpochUnavailable("recovery authority path is not canonical") + return path + + +def read_bound_file(bound: RecoveryFile) -> bytes: + """Read the original protected file, retaining its actual inode and bytes.""" + path = _path(bound.path) + _approved_directory(path.parent, uid=os.geteuid()) + data, identity = read_original(path) + if identity != (bound.device, bound.inode) or hashlib.sha256(data).hexdigest() != bound.sha256: + raise EpochUnavailable("recovery original file changed: " + bound.path) + return data + + +def _load_authorization(path: Path, inventory: EpochInventory) -> AuthorizationObservation: + """Explicit component seam; the production factory uses one fixed path.""" + if inventory.schema_version != 2 or not inventory.authorization_sha256: + raise EpochUnavailable("complete issuer requires the fixed version-two inventory") + data, (device, inode) = read_original(path) + digest = hashlib.sha256(data).hexdigest() + if digest != inventory.authorization_sha256: + raise EpochUnavailable("recovery authorization differs from fixed inventory") + raw = json.loads(data, object_pairs_hook=unique_pairs) + if canonical(raw) != data: + raise EpochUnavailable("recovery authorization is not canonical") + authorization = EpochAuthorization.parse(raw) + for field in ("deployment_id", "authorization_id", "request_id", "preparation_id", "target_id"): + if getattr(authorization, field) != getattr(inventory, field): + raise EpochUnavailable("recovery authorization differs from inventory: " + field) + if inventory.enrollment_ids != (authorization.enrollment_id,): + raise EpochUnavailable("recovery enrollment differs from complete fixed scope") + return AuthorizationObservation(authorization, RecoveryFile(str(path), digest, device, inode)) + + +def load_authorization(inventory: EpochInventory) -> AuthorizationObservation: + """Read the deployment-fixed authorization; no CLI or environment root.""" + _approved_directory(AUTHORIZATION_ROOT, uid=0) + result = _load_authorization(AUTHORIZATION_FILE, inventory) + check_authorization(result, inventory) + return result + + +def check_authorization(previous: AuthorizationObservation, inventory: EpochInventory) -> None: + if _load_authorization(Path(previous.original.path), inventory) != previous: + raise EpochUnavailable("recovery authorization changed after observation") + if Path(previous.original.path) != AUTHORIZATION_FILE: + raise EpochUnavailable("recovery authorization did not use the fixed production path") + _approved_directory(PEER_ROOT, uid=0) + peer = PEER_ROOT.stat() + if (peer.st_dev, peer.st_ino) != (previous.authorization.socket_directory_device, previous.authorization.socket_directory_inode): + raise EpochUnavailable("fixed recovery peer directory was replaced") + + +def _same_path_version_mount(enrollment: RecoveryEnrollment, config: BootstrapConfig) -> None: + version = config.management_version_root + if version is None: + raise EpochUnavailable("recovery target has no installed runtime root") + selected = [mount for mount in enrollment.target.mounts if mount.container_id == enrollment.target.management_id and mount.destination == str(version)] + if len(selected) != 1 or selected[0].source != str(version) or selected[0].writable: + raise EpochUnavailable("recovery READY requires its exact original same-path read-only version mount") + + +def observe_target_ready(authorization: EpochAuthorization, enrollment: RecoveryEnrollment) -> TargetReadyObservation: + """Invoke the actual stager verifier against the retained installed READY.""" + target = enrollment.target + if (enrollment.deployment_id, enrollment.authorization_id, enrollment.enrollment_id, target.management_id) != (authorization.deployment_id, authorization.authorization_id, authorization.enrollment_id, authorization.target_id): + raise EpochUnavailable("current target enrollment differs from recovery authorization") + read_bound_file(target.bootstrap_config) + read_bound_file(target.management_config) + read_bound_file(authorization.consumer_config) + ready_bytes = read_bound_file(authorization.ready_receipt) + config = BootstrapConfig.read(Path(target.bootstrap_config.path)) + if config.instance_id != target.instance_id or config.management_config is None or config.management_version_root is None or config.recovery_epoch_config != Path("/bootstrap/recovery-epoch.json"): + raise EpochUnavailable("exact recovery bootstrap profile is unavailable") + _same_path_version_mount(enrollment, config) + mounts = {mount.destination: mount for mount in target.mounts} + consumer_mount = mounts.get("/bootstrap/recovery-epoch.json") + if consumer_mount is None or consumer_mount.source != authorization.consumer_config.path or consumer_mount.writable: + raise EpochUnavailable("target recovery consumer configuration mount differs") + runtime = next(item for item in config.runtimes if item.reference == config.initial_runtime) + if runtime.execution_kind != "management-installed" or runtime.ready_receipt != Path(authorization.ready_receipt.path): + raise EpochUnavailable("recovery target requires the authorized original installed READY") + # Host verification and the exact same-path RO mount must refer to the + # same complete interpreter distribution. A host-only symlink target can + # otherwise produce valid host READY while the Docker child is unusable. + resolved_interpreter = runtime.executable.resolve(strict=True) + version_root = config.management_version_root.resolve(strict=True) + if not resolved_interpreter.is_relative_to(version_root) or resolved_interpreter != Path(authorization.python_executable).resolve(strict=True): + raise EpochUnavailable("recovery interpreter escapes the exactly mounted runtime closure") + safe_python = {"PYTHONDONTWRITEBYTECODE": "1", "PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + if any(name.startswith("LD_") or (name.startswith("PYTHON") and safe_python.get(name) != value) for name, value in runtime.environment): + raise EpochUnavailable("recovery target runtime contains unsupported executable hooks") + authority = _source_authority(Path(authorization.source_token_file)) + staging = ManagementVersionStager( + ManagementRuntimeConfig( + config.instance_id, + MANAGEMENT_REPOSITORY, + Path(authorization.source_repository), + config.management_version_root, + Path(authorization.python_executable), + Path(authorization.uv_executable), + authorization.uv_version, + protected_roots=(config.state_root, config.oauth_root, config.journal_root), + management_config=Path(target.management_config.path), + source_authority=authority, + ) + ) + receipt = staging.lookup(Path(authorization.ready_receipt.path)) + if receipt.source_sha != authorization.source_sha or receipt.instance_id != target.instance_id or json.loads(ready_bytes, object_pairs_hook=unique_pairs) != asdict(receipt): + raise EpochUnavailable("genuine installed READY differs from exact authorization") + if Path(receipt.environment_path).parent.name != authorization.preparation_id: + raise EpochUnavailable("installed runtime belongs to another exact preparation") + expected = runtime_for_receipt(config, receipt) + if runtime != expected: + raise EpochUnavailable("target invocation differs from its original installed runtime") + for original in (target.bootstrap_config, target.management_config, authorization.consumer_config, authorization.ready_receipt): + read_bound_file(original) + runtime_bytes = json.dumps(asdict(runtime), sort_keys=True, separators=(",", ":"), default=str, allow_nan=False).encode() + return TargetReadyObservation(receipt, target.bootstrap_config, target.management_config, authorization.consumer_config, authorization.ready_receipt, hashlib.sha256(runtime_bytes).hexdigest()) + + +def verify_target_ready(authorization: EpochAuthorization, enrollment: RecoveryEnrollment, previous: TargetReadyObservation) -> TargetReadyObservation: + current = observe_target_ready(authorization, enrollment) + if current != previous: + raise EpochUnavailable("target source, configuration or READY changed before grant") + return current diff --git a/src/auto_coder/management/recovery_epoch_bootstrap.py b/src/auto_coder/management/recovery_epoch_bootstrap.py new file mode 100644 index 000000000..4706c5ed7 --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_bootstrap.py @@ -0,0 +1,83 @@ +"""Explicit bootstrap custody endpoint for the one-use recovery issuer. + +This stage owns no child launch, durable consumer launch claim, application +readiness or operation admission. Those belong to the separate epoch consumer. +""" + +from __future__ import annotations + +import fcntl +import os +import signal +import stat +import sys +import threading +from pathlib import Path +from typing import TYPE_CHECKING + +from auto_coder.logger_config import get_logger, setup_logger + +from .original_attempts import identity +from .recovery_epoch_consumer import CONSUMER_CONFIG, connect_recovery_epoch + +if TYPE_CHECKING: + from .bootstrap_supervisor import BootstrapConfig + +logger = get_logger(__name__) + + +class RecoveryBootstrapCancelled(ValueError): + """Terminate custody immediately; this endpoint never owns a live child.""" + + +def _open_target_owner(config: BootstrapConfig) -> int: + config.validate() + if config.recovery_epoch_config != CONSUMER_CONFIG: + raise ValueError("recovery custody requires its fixed selected bootstrap profile") + identity(config.state_root, directory=True) + # The separately authorized preparation must already provide the exact + # target file. This path cannot create or repair an old or missing owner. + path = config.state_root / ".supervisor-owner.lock" + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + actual = os.fstat(descriptor) + if identity(path) != (actual.st_dev, actual.st_ino) or not stat.S_ISREG(actual.st_mode) or stat.S_IMODE(actual.st_mode) != 0o600 or actual.st_uid != os.geteuid() or actual.st_nlink != 1: + raise ValueError("recovery target owner is not the protected original file") + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + if set(os.listdir(config.state_root)) - {".supervisor-owner.lock", "logs"} or os.listdir(config.journal_root): + raise ValueError("recovery target contains prior or uncertain launch state") + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def run_recovery_epoch_bootstrap(config: BootstrapConfig) -> int: + """Retain acknowledged fences while the authenticated host remains live.""" + owner = -1 + pause = threading.Event() + previous_handlers: dict[int, object] = {} + try: + owner = _open_target_owner(config) + setup_logger(stream=sys.stdout, log_file=str(config.state_root / "logs" / "recovery-epoch-bootstrap.log")) + + def cancel(*_: object) -> None: + raise RecoveryBootstrapCancelled("recovery custody was cancelled") + + for signum in (signal.SIGTERM, signal.SIGINT): + previous_handlers[signum] = signal.signal(signum, cancel) + with connect_recovery_epoch(CONSUMER_CONFIG, owner, timeout=config.verification_timeout) as consumer: + logger.bind(recovery_epoch_phase="consumer-custody", current_ready=False, child_started=False).info("Recovery custody acknowledged; no child launch or readiness is established") + while True: + consumer.check_current() + pause.wait(0.2) + except (OSError, ValueError, TypeError, KeyError) as error: + logger.bind(recovery_epoch_phase="consumer-unavailable", current_ready=False, child_started=False).error("Recovery custody unavailable: {}", error) + return 1 + finally: + for saved_signal, handler in previous_handlers.items(): + signal.signal(saved_signal, handler) # type: ignore[arg-type] + if owner >= 0: + # This endpoint never launches a child, so it has no live child + # whose containment could outlast these retained fences. + os.close(owner) diff --git a/src/auto_coder/management/recovery_epoch_consumer.py b/src/auto_coder/management/recovery_epoch_consumer.py new file mode 100644 index 000000000..779d4324a --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_consumer.py @@ -0,0 +1,414 @@ +"""Retain recovery issuer/consumer custody before any child launch authority. + +This helper does not claim a child, create stores, launch, or declare readiness. +The owning Supervisor must retain the returned handle until it can safely end +custody. A current-host failure never implicitly releases already received FDs. +""" + +from __future__ import annotations + +import fcntl +import json +import os +import re +import secrets +import select +import socket +import stat +import struct +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, cast + +from .archived_exclusions import unique_pairs +from .original_attempts import read_original +from .recovery_epoch_peer import CHALLENGE_SCHEMA, MAX_MESSAGE_BYTES, MAX_RESOURCES, OFFER_SCHEMA, RECEIPT_SCHEMA, HeldFence, KernelLock, PeerUnavailable, ResourceIdentity, _name, _reject_constant, _token, encode_peer_message, inspect_held_resource, inspect_kernel_lock, receive_peer_message + +CONSUMER_CONFIG = Path("/bootstrap/recovery-epoch.json") +ISSUER_DIRECTORY = Path("/management-recovery") +ISSUER_SOCKET = ISSUER_DIRECTORY / "issuer.sock" +GRANT_SCHEMA = "management-recovery-epoch-grant-v1" +# Linux UAPI asm-generic/socket.h; unavailable kernels refuse without a fallback. +_SO_PEERPIDFD = getattr(socket, "SO_PEERPIDFD", 77) + + +@dataclass(frozen=True) +class OriginalFence: + name: str + path: str + device: int + inode: int + + +@dataclass(frozen=True) +class TargetOwner: + name: str + device: int + inode: int + + +@dataclass(frozen=True) +class TransferredResource: + name: str + kind: Literal["transferred-ex", "network-namespace"] + device: int + inode: int + + +@dataclass(frozen=True) +class ConsumerConfiguration: + schema_version: int + request_id: str + authorization_id: str + target_id: str + issuer_uid: int + socket_directory_device: int + socket_directory_inode: int + original_fences: tuple[OriginalFence, ...] + target_owner: TargetOwner + transferred_resources: tuple[TransferredResource, ...] + + +@dataclass(frozen=True) +class RecoveryGrant: + request_id: str + authorization_id: str + target_id: str + challenge: str + receipt_nonce: str + inventory_sha256: str + binding_sha256: str + consumed_sha256: str + receipt_sha256: str + host_boot_id: str + host_pid: int + host_start_token: str + + +@dataclass(frozen=True) +class ConsumerTransfer: + expected: TransferredResource + descriptor: int + mode: int + uid: int + link: str + lock: KernelLock | None + + +def _identity(value: object) -> bool: + return type(value) is int and value > 0 + + +def parse_consumer_configuration(encoded: bytes) -> ConsumerConfiguration: + """Parse a complete pinned target configuration, not host authorization.""" + if len(encoded) > MAX_MESSAGE_BYTES: + raise PeerUnavailable("consumer configuration exceeds its bound") + value = json.loads(encoded, object_pairs_hook=unique_pairs, parse_constant=_reject_constant) + if not isinstance(value, dict) or set(value) != set(ConsumerConfiguration.__dataclass_fields__): + raise PeerUnavailable("consumer configuration schema differs") + if type(value["schema_version"]) is not int or value["schema_version"] != 1 or type(value["issuer_uid"]) is not int or value["issuer_uid"] < 0: + raise PeerUnavailable("consumer configuration version or issuer differs") + if not all(_name(value[name]) for name in ("request_id", "authorization_id", "target_id")) or not all(_identity(value[name]) for name in ("socket_directory_device", "socket_directory_inode")): + raise PeerUnavailable("consumer binding or socket directory identity is invalid") + originals, transferred, target = value["original_fences"], value["transferred_resources"], value["target_owner"] + if not isinstance(originals, list) or not originals or not isinstance(transferred, list) or not transferred or len(originals) + len(transferred) + 1 > MAX_RESOURCES: + raise PeerUnavailable("consumer resource set is incomplete or oversized") + if not isinstance(target, dict) or set(target) != set(TargetOwner.__dataclass_fields__): + raise PeerUnavailable("consumer target owner binding is incomplete") + for item in originals: + if not isinstance(item, dict) or set(item) != set(OriginalFence.__dataclass_fields__) or not isinstance(item.get("path"), str) or not Path(item["path"]).is_absolute() or "\x00" in item["path"]: + raise PeerUnavailable("consumer original fence binding is incomplete") + for item in transferred: + if not isinstance(item, dict) or set(item) != set(TransferredResource.__dataclass_fields__) or item.get("kind") not in ("transferred-ex", "network-namespace"): + raise PeerUnavailable("consumer transferred resource binding is incomplete") + entries = [*originals, target, *transferred] + if any(not _name(item["name"]) or not _identity(item["device"]) or not _identity(item["inode"]) for item in entries): + raise PeerUnavailable("consumer resource identity is invalid") + if len({item["name"] for item in entries}) != len(entries) or len({(item["device"], item["inode"]) for item in entries}) != len(entries) or len({item["path"] for item in originals}) != len(originals): + raise PeerUnavailable("consumer resource aliases are conflicting") + if not any(item["kind"] == "transferred-ex" for item in transferred): + raise PeerUnavailable("consumer deployment/controller exclusive custody is missing") + return ConsumerConfiguration( + value["schema_version"], + value["request_id"], + value["authorization_id"], + value["target_id"], + value["issuer_uid"], + value["socket_directory_device"], + value["socket_directory_inode"], + tuple(OriginalFence(**item) for item in originals), + TargetOwner(**target), + tuple(TransferredResource(**item) for item in transferred), + ) + + +def _acquire_originals(configuration: ConsumerConfiguration) -> tuple[HeldFence, ...]: + fences: list[HeldFence] = [] + try: + for expected in configuration.original_fences: + path = Path(expected.path) + if path != path.resolve(strict=True): + raise PeerUnavailable("consumer original fence path is not canonical") + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + fences.append(HeldFence(expected.name, descriptor, "original-sh")) + actual = os.fstat(descriptor) + if (actual.st_dev, actual.st_ino) != (expected.device, expected.inode): + raise PeerUnavailable("consumer original fence object differs") + fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) + inspect_held_resource(fences[-1]) + return tuple(fences) + except BaseException: + for fence in fences: + os.close(fence.descriptor) + raise + + +def _target(configuration: ConsumerConfiguration, descriptor: int) -> HeldFence: + # Using the independent EX inspection also rejects an inherited target lock. + fence = HeldFence(configuration.target_owner.name, descriptor, "transferred-ex") + actual = inspect_held_resource(fence) + if (actual.device, actual.inode) != (configuration.target_owner.device, configuration.target_owner.inode): + raise PeerUnavailable("consumer target owner differs from the pinned configuration") + return fence + + +def _transfer(expected: TransferredResource, descriptor: int, issuer_pid: int, issuer_uid: int) -> ConsumerTransfer: + info = os.fstat(descriptor) + link = os.readlink(f"/proc/self/fd/{descriptor}") + if (info.st_dev, info.st_ino) != (expected.device, expected.inode) or os.get_inheritable(descriptor): + raise PeerUnavailable("received resource identity or close-on-exec flag differs") + if expected.kind == "network-namespace": + if link != f"net:[{info.st_ino}]": + raise PeerUnavailable("received resource is not the pinned network namespace") + lock = None + else: + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or info.st_uid != issuer_uid: + raise PeerUnavailable("received exclusive fence protection differs") + # A host outside the consumer PID namespace maps to PID 0 in fdinfo. + # SO_PEERPIDFD, obtained independently from this socket, pins that peer. + lock = inspect_kernel_lock(os.getpid(), descriptor, info.st_dev, info.st_ino, allow_unmapped_owner=issuer_pid == 0) + if lock is None or lock.mode != "WRITE" or lock.pid != issuer_pid: + raise PeerUnavailable("received exclusive fence is not held by the authenticated issuer") + return ConsumerTransfer(expected, descriptor, info.st_mode, info.st_uid, link, lock) + + +class RecoveryEpochConsumer: + """Retained prelaunch custody; the owner alone decides when it may close.""" + + def __init__(self, connection: socket.socket, configuration: ConsumerConfiguration, target_owner_fd: int, *, timeout: float = 30, configuration_file: Path | None = None) -> None: + self.connection = connection + self.configuration = configuration + self.grant: RecoveryGrant | None = None + self.challenge = secrets.token_hex(32) + self.receipt_nonce = "" + self._timeout = timeout + self._configuration_file = configuration_file + self._configuration_bytes: bytes | None = None + self._configuration_identity: tuple[int, int] | None = None + self._issuer_pidfd = -1 + self._issuer_pid = -1 + self._closed = False + self._originals: tuple[HeldFence, ...] = () + self._target: HeldFence | None = None + self._owned: list[int] = [] + self._initial: tuple[ResourceIdentity, ...] = () + self._transfers: tuple[ConsumerTransfer, ...] = () + try: + if type(timeout) not in (int, float) or not 0 < timeout <= 3600 or connection.family != socket.AF_UNIX or connection.type not in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET): + raise PeerUnavailable("consumer requires a bounded authenticated local exchange") + if configuration_file is not None: + self._configuration_bytes, self._configuration_identity = read_original(configuration_file) + if parse_consumer_configuration(self._configuration_bytes) != configuration: + raise PeerUnavailable("consumer configuration changed before connection") + self._target = _target(configuration, target_owner_fd) + self._originals = _acquire_originals(configuration) + self._initial = tuple(inspect_held_resource(fence) for fence in (*self._originals, self._target)) + self._issuer_pid, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))) + if uid != configuration.issuer_uid or self._issuer_pid < 0: + raise PeerUnavailable("connected issuer credentials differ") + self._issuer_pidfd = connection.getsockopt(socket.SOL_SOCKET, _SO_PEERPIDFD) + os.set_inheritable(self._issuer_pidfd, False) + if os.readlink(f"/proc/self/fd/{self._issuer_pidfd}") != "anon_inode:[pidfd]": + raise PeerUnavailable("connected issuer has no kernel peer lifetime pin") + self._check_issuer() + self._exchange() + except BaseException as error: + self.close() + if isinstance(error, (OSError, ValueError, TypeError, KeyError)) and not isinstance(error, PeerUnavailable): + raise PeerUnavailable("recovery consumer custody is unavailable") from error + raise + + def __enter__(self) -> RecoveryEpochConsumer: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + @property + def issuer_pidfd(self) -> int: + return self._issuer_pidfd + + @property + def retained_descriptors(self) -> tuple[int, ...]: + return (*[fence.descriptor for fence in self._originals], *self._owned) + + def close(self) -> None: + """Release only when the owning lifecycle has established it is safe.""" + if not self._closed: + self._closed = True + self.connection.close() + for descriptor in self.retained_descriptors: + os.close(descriptor) + self._originals = () + self._owned.clear() + if self._issuer_pidfd >= 0: + os.close(self._issuer_pidfd) + self._issuer_pidfd = -1 + + def _check_issuer(self) -> None: + if self._closed or self._issuer_pidfd < 0: + raise PeerUnavailable("issuer custody handle is closed") + pid, uid, _ = struct.unpack("3i", self.connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))) + poll = select.poll() + poll.register(self._issuer_pidfd, select.POLLIN) + if (pid, uid) != (self._issuer_pid, self.configuration.issuer_uid) or poll.poll(0): + raise PeerUnavailable("the authenticated issuer lifetime ended") + channel = select.poll() + channel.register(self.connection, select.POLLERR | select.POLLHUP | select.POLLRDHUP) + if channel.poll(0): + raise PeerUnavailable("the authenticated issuer channel ended") + + def _binding(self) -> dict[str, object]: + return {"request_id": self.configuration.request_id, "authorization_id": self.configuration.authorization_id, "target_id": self.configuration.target_id, "challenge": self.challenge} + + def _exchange(self) -> None: + deadline = time.monotonic() + self._timeout + self.connection.settimeout(min(self._timeout, 60)) + self.connection.sendall(encode_peer_message({"schema": CHALLENGE_SCHEMA, **self._binding()})) + offer, descriptors = receive_peer_message(self.connection, timeout=self._timeout, allow_rights=True, deadline=deadline) + self._owned.extend(descriptors) + if set(offer) != {"schema", *self._binding(), "receipt_nonce", "resources"} or offer["schema"] != OFFER_SCHEMA or any(offer[name] != value for name, value in self._binding().items()) or not _token(offer["receipt_nonce"]): + raise PeerUnavailable("issuer custody offer is stale or bound to another target") + self.receipt_nonce = str(offer["receipt_nonce"]) + resources = offer["resources"] + expected: dict[str, tuple[str, int, int]] = {fence.name: ("original-sh", fence.device, fence.inode) for fence in self.configuration.original_fences} + owner = self.configuration.target_owner + expected[owner.name] = ("target-ex", owner.device, owner.inode) + expected.update({resource.name: (resource.kind, resource.device, resource.inode) for resource in self.configuration.transferred_resources}) + if not isinstance(resources, list) or len(resources) != len(expected): + raise PeerUnavailable("issuer omitted required custody resources") + seen: set[str] = set() + transferred: dict[str, int] = {} + for resource in resources: + if not isinstance(resource, dict) or set(resource) != {"name", "kind", "device", "inode", "transfer_index"} or not isinstance(resource["name"], str): + raise PeerUnavailable("issuer custody resource schema differs") + name = resource["name"] + if name in seen or name not in expected or (resource["kind"], resource["device"], resource["inode"]) != expected[name] or type(resource["device"]) is not int or type(resource["inode"]) is not int: + raise PeerUnavailable("issuer custody resource differs from the pinned configuration") + seen.add(name) + index = resource["transfer_index"] + if resource["kind"] in ("transferred-ex", "network-namespace"): + if type(index) is not int or not 0 <= index < len(descriptors) or index in transferred.values(): + raise PeerUnavailable("issuer transfer descriptor mapping is incomplete or repeated") + transferred[name] = index + elif index is not None: + raise PeerUnavailable("original shared or target exclusive custody must be independently acquired") + if seen != set(expected) or set(transferred.values()) != set(range(len(descriptors))) or len(descriptors) != len(self.configuration.transferred_resources): + raise PeerUnavailable("issuer delivered an incomplete or unexpected descriptor set") + self._transfers = tuple(_transfer(resource, descriptors[transferred[resource.name]], self._issuer_pid, self.configuration.issuer_uid) for resource in self.configuration.transferred_resources) + self.check_current() + assert self._target is not None + received = [{"name": fence.name, "fd": fence.descriptor} for fence in (*self._originals, self._target)] + received.extend({"name": item.expected.name, "fd": item.descriptor} for item in self._transfers) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PeerUnavailable("consumer verification exceeded its bounded deadline") + self.connection.settimeout(min(remaining, 60)) + self.connection.sendall(encode_peer_message({"schema": RECEIPT_SCHEMA, **self._binding(), "receipt_nonce": self.receipt_nonce, "descriptors": received})) + grant, _ = receive_peer_message(self.connection, timeout=self._timeout, deadline=deadline) + expected_fields = {"schema", "granted", *RecoveryGrant.__dataclass_fields__} + if set(grant) != expected_fields or grant["schema"] != GRANT_SCHEMA or grant["granted"] != "recovery-epoch-consumed" or any(grant[name] != value for name, value in self._binding().items()) or grant["receipt_nonce"] != self.receipt_nonce: + raise PeerUnavailable("issuer grant is not the exact current consumed challenge") + if not all(_token(grant[name]) for name in ("inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256")): + raise PeerUnavailable("issuer grant durable identities are incomplete") + boot, pid, started = grant["host_boot_id"], grant["host_pid"], grant["host_start_token"] + if not isinstance(boot, str) or str(uuid.UUID(boot)) != boot or boot != Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() or not _identity(pid) or not isinstance(started, str) or re.fullmatch(r"[1-9][0-9]*", started) is None: + raise PeerUnavailable("issuer grant original lifetime is malformed or from another boot") + if self._issuer_pid > 0: + fields = Path(f"/proc/{self._issuer_pid}/stat").read_text(encoding="ascii").rsplit(")", 1)[1].split() + if pid != self._issuer_pid or len(fields) < 20 or fields[19] != started or fields[0] in ("Z", "X", "x"): + raise PeerUnavailable("issuer grant differs from its currently visible lifetime") + self.check_current() + self.grant = RecoveryGrant( + self.configuration.request_id, + self.configuration.authorization_id, + self.configuration.target_id, + self.challenge, + self.receipt_nonce, + cast(str, grant["inventory_sha256"]), + cast(str, grant["binding_sha256"]), + cast(str, grant["consumed_sha256"]), + cast(str, grant["receipt_sha256"]), + boot, + cast(int, pid), + started, + ) + + def check_current(self) -> None: + """Refuse stale readiness without releasing any already retained custody.""" + try: + self._check_issuer() + assert self._target is not None + if tuple(inspect_held_resource(fence) for fence in (*self._originals, self._target)) != self._initial: + raise PeerUnavailable("consumer independent fence custody changed") + if tuple(_transfer(item.expected, item.descriptor, self._issuer_pid, self.configuration.issuer_uid) for item in self._transfers) != self._transfers: + raise PeerUnavailable("consumer transferred custody changed") + if self._configuration_file is not None: + current, identity = read_original(self._configuration_file) + if current != self._configuration_bytes or identity != self._configuration_identity: + raise PeerUnavailable("consumer configuration changed while holding custody") + self._check_issuer() + except (OSError, ValueError, TypeError, KeyError) as error: + if isinstance(error, PeerUnavailable): + raise + raise PeerUnavailable("current consumer custody could not be verified") from error + + +def _validate_socket_directory(configuration: ConsumerConfiguration) -> None: + if ISSUER_DIRECTORY != ISSUER_DIRECTORY.resolve(strict=True): + raise PeerUnavailable("recovery issuer directory is not canonical") + directory = os.open(ISSUER_DIRECTORY, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(directory) + if (info.st_dev, info.st_ino) != (configuration.socket_directory_device, configuration.socket_directory_inode) or info.st_uid != configuration.issuer_uid or stat.S_IMODE(info.st_mode) != 0o700: + raise PeerUnavailable("recovery issuer directory differs from the target configuration") + entry = ISSUER_SOCKET.lstat() + if not stat.S_ISSOCK(entry.st_mode) or entry.st_uid != configuration.issuer_uid or stat.S_IMODE(entry.st_mode) != 0o600: + raise PeerUnavailable("recovery issuer socket protection differs") + current = ISSUER_DIRECTORY.lstat() + if (current.st_dev, current.st_ino, current.st_uid, current.st_mode) != (info.st_dev, info.st_ino, info.st_uid, info.st_mode): + raise PeerUnavailable("recovery issuer directory changed during inspection") + finally: + os.close(directory) + + +def connect_recovery_epoch(config_path: Path, target_owner_fd: int, *, timeout: float = 30) -> RecoveryEpochConsumer: + """Connect only the fixed production origin; no request or environment root.""" + if config_path != CONSUMER_CONFIG: + raise PeerUnavailable("recovery consumer configuration must use its fixed bootstrap path") + if type(timeout) not in (int, float) or not 0 < timeout <= 3600: + raise PeerUnavailable("recovery consumer verification deadline must be positive and at most one hour") + encoded, _ = read_original(config_path) + configuration = parse_consumer_configuration(encoded) + _validate_socket_directory(configuration) + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + connection.settimeout(min(timeout, 60)) + connection.connect(str(ISSUER_SOCKET)) + _validate_socket_directory(configuration) + return RecoveryEpochConsumer(connection, configuration, target_owner_fd, timeout=timeout, configuration_file=config_path) + except BaseException: + connection.close() + raise diff --git a/src/auto_coder/management/recovery_epoch_peer.py b/src/auto_coder/management/recovery_epoch_peer.py new file mode 100644 index 000000000..f2e6fd665 --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_peer.py @@ -0,0 +1,415 @@ +"""Authenticated recovery peer custody evidence, without grant authority. + +The issuer supplies the expected Docker-bound process and the complete fence +set. This transport neither decides that scope nor consumes or issues an epoch. +Caller-owned fences stay open; the session owns only its socket and PID pin. +""" + +from __future__ import annotations + +import array +import json +import os +import re +import secrets +import select +import socket +import stat +import struct +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from .archived_exclusions import unique_pairs +from .fresh_host_observation import _assert_pid_flock + +MAX_MESSAGE_BYTES = 64 * 1024 +MAX_RESOURCES = 64 +CHALLENGE_SCHEMA = "management-recovery-peer-challenge-v1" +OFFER_SCHEMA = "management-recovery-custody-offer-v1" +RECEIPT_SCHEMA = "management-recovery-custody-receipt-v1" +FenceKind = Literal["original-sh", "target-ex", "transferred-ex"] + + +class PeerUnavailable(ValueError): + """Peer custody is unavailable; this result never permits a grant.""" + + +@dataclass(frozen=True) +class ExpectedPeer: + pid: int + start_token: str + boot_id: str + uid: int + request_id: str + authorization_id: str + target_id: str + + +@dataclass(frozen=True) +class HeldFence: + name: str + descriptor: int + kind: FenceKind + + +@dataclass(frozen=True) +class HeldResource: + name: str + descriptor: int + kind: Literal["network-namespace"] = "network-namespace" + + +@dataclass(frozen=True) +class KernelLock: + mode: str + pid: int + device_major: int + device_minor: int + inode: int + + +@dataclass(frozen=True) +class ResourceIdentity: + name: str + kind: str + path: str + device: int + inode: int + mode: int + uid: int + lock: KernelLock | None + + +@dataclass(frozen=True) +class ReceivedResource: + identity: ResourceIdentity + peer_descriptor: int + peer_lock: KernelLock | None + + +@dataclass(frozen=True) +class PeerCustody: + peer: ExpectedPeer + challenge: str + receipt_nonce: str + resources: tuple[ReceivedResource, ...] + observed_monotonic_ns: int + + +def _token(value: object) -> bool: + return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None + + +def _name(value: object) -> bool: + return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}", value) is not None + + +def _fd_path(pid: int, descriptor: int, *, info: bool = False) -> Path: + if type(pid) is not int or pid <= 0 or type(descriptor) is not int or not 0 <= descriptor < 1_048_576: + raise PeerUnavailable("peer descriptor or process identity is invalid") + return Path("/proc") / str(pid) / ("fdinfo" if info else "fd") / str(descriptor) + + +def inspect_kernel_lock(pid: int, descriptor: int, device: int, inode: int, *, allow_unmapped_owner: bool = False) -> KernelLock | None: + """Read the exact FD's normalized kernel flock record without changing it. + + The optional PID-zero allowance is only for an authenticated host outside + a consumer's PID namespace; independent holder checks must use the default. + """ + path = _fd_path(pid, descriptor, info=True) + with path.open("rb") as stream: + data = stream.read(MAX_MESSAGE_BYTES + 1) + if len(data) > MAX_MESSAGE_BYTES: + raise PeerUnavailable("kernel descriptor observation exceeds its bound") + matches: list[KernelLock] = [] + for line in data.decode("ascii").splitlines(): + if not line.startswith("lock:"): + continue + fields = line.removeprefix("lock:").split() + if len(fields) != 8 or fields[1:3] != ["FLOCK", "ADVISORY"] or fields[3] not in ("READ", "WRITE") or fields[6:] != ["0", "EOF"]: + raise PeerUnavailable("descriptor kernel lock profile differs") + parts = fields[5].split(":") + if len(parts) != 3 or not fields[4].isdecimal() or int(fields[4]) < (0 if allow_unmapped_owner else 1): + raise PeerUnavailable("descriptor kernel lock identity is malformed") + observed = KernelLock(fields[3], int(fields[4]), int(parts[0], 16), int(parts[1], 16), int(parts[2])) + if (observed.device_major, observed.device_minor, observed.inode) != (os.major(device), os.minor(device), inode): + raise PeerUnavailable("descriptor kernel lock names another file") + matches.append(observed) + if len(matches) > 1: + raise PeerUnavailable("descriptor kernel lock is ambiguous") + return matches[0] if matches else None + + +def inspect_held_resource(resource: HeldFence | HeldResource) -> ResourceIdentity: + """Inspect a caller-owned descriptor without acquiring or releasing it.""" + if not _name(resource.name): + raise PeerUnavailable("resource name is invalid") + descriptor_path = _fd_path(os.getpid(), resource.descriptor) + info = os.fstat(resource.descriptor) + path = os.readlink(descriptor_path) + if isinstance(resource, HeldResource): + if resource.kind != "network-namespace" or re.fullmatch(r"net:\[[0-9]+\]", path) is None or path != f"net:[{info.st_ino}]": + raise PeerUnavailable("held resource is not the exact network namespace") + return ResourceIdentity(resource.name, resource.kind, path, info.st_dev, info.st_ino, info.st_mode, info.st_uid, None) + if resource.kind not in ("original-sh", "target-ex", "transferred-ex"): + raise PeerUnavailable("held fence kind is unsupported") + original = Path(path) + if not original.is_absolute() or original != original.resolve(strict=True): + raise PeerUnavailable("held fence path is not canonical") + current = original.lstat() + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or info.st_uid != os.geteuid() or (current.st_dev, current.st_ino, current.st_mode, current.st_uid, current.st_nlink) != (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_nlink): + raise PeerUnavailable("held fence identity or protection differs") + lock = inspect_kernel_lock(os.getpid(), resource.descriptor, info.st_dev, info.st_ino) + if resource.kind != "target-ex": + mode = "READ" if resource.kind == "original-sh" else "WRITE" + if lock is None or lock.mode != mode or lock.pid != os.getpid(): + raise PeerUnavailable("issuer does not independently hold the required fence") + _assert_pid_flock(os.getpid(), resource.descriptor, mode) + return ResourceIdentity(resource.name, resource.kind, path, info.st_dev, info.st_ino, info.st_mode, info.st_uid, lock) + + +def _received(identity: ResourceIdentity, peer: ExpectedPeer, descriptor: int) -> ReceivedResource: + path = _fd_path(peer.pid, descriptor) + before = path.stat() + # /proc/PID/fd intentionally follows the authenticated process's descriptor. + # NONBLOCK prevents a concurrently replaced FIFO from blocking inspection. + reopened = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK) + try: + actual = os.fstat(reopened) + after = path.stat() + expected = (identity.device, identity.inode, identity.mode, identity.uid) + for info in (before, actual, after): + if (info.st_dev, info.st_ino, info.st_mode, info.st_uid) != expected: + raise PeerUnavailable("received descriptor names another resource") + if identity.kind == "network-namespace": + if os.readlink(path) != identity.path: + raise PeerUnavailable("received network namespace object differs") + observed = None + else: + observed = inspect_kernel_lock(peer.pid, descriptor, actual.st_dev, actual.st_ino) + if identity.kind == "transferred-ex": + if observed is None or observed != identity.lock or observed.mode != "WRITE": + raise PeerUnavailable("consumer did not receive the held exclusive open file description") + else: + mode = "READ" if identity.kind == "original-sh" else "WRITE" + if observed is None or observed.pid != peer.pid or observed.mode != mode: + raise PeerUnavailable("consumer did not independently acquire the required fence") + _assert_pid_flock(peer.pid, reopened, mode) + final = path.stat() + if (final.st_dev, final.st_ino, final.st_mode, final.st_uid) != expected: + raise PeerUnavailable("consumer descriptor changed during custody observation") + return ReceivedResource(identity, descriptor, observed) + finally: + os.close(reopened) + + +def _reject_constant(value: str) -> object: + raise PeerUnavailable("non-finite peer JSON value: " + value) + + +def encode_peer_message(value: dict[str, object]) -> bytes: + """Encode one bounded protocol object; this does not authorize its content.""" + packet = json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + b"\n" + if len(packet) > MAX_MESSAGE_BYTES: + raise PeerUnavailable("peer message exceeds its bound") + return packet + + +def receive_peer_message(connection: socket.socket, *, timeout: float, allow_rights: bool = False, deadline: float | None = None) -> tuple[dict[str, object], tuple[int, ...]]: + """Read one message, closing all kernel-delivered FDs on any refusal.""" + deadline = time.monotonic() + timeout if deadline is None else deadline + content = bytearray() + received: list[int] = [] + try: + while b"\n" not in content: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PeerUnavailable("peer exchange exceeded its bounded deadline") + connection.settimeout(remaining) + data, ancillary, flags, _ = connection.recvmsg(MAX_MESSAGE_BYTES + 1, socket.CMSG_SPACE(MAX_RESOURCES * array.array("i").itemsize), socket.MSG_CMSG_CLOEXEC) + for level, kind, payload in ancillary: + if level == socket.SOL_SOCKET and kind == socket.SCM_RIGHTS: + values = array.array("i") + values.frombytes(payload[: len(payload) - len(payload) % values.itemsize]) + received.extend(values) + if (ancillary and not allow_rights) or any((level, kind) != (socket.SOL_SOCKET, socket.SCM_RIGHTS) for level, kind, _ in ancillary) or flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC): + raise PeerUnavailable("peer unexpectedly supplied descriptors or a truncated message") + if len(received) > MAX_RESOURCES: + raise PeerUnavailable("peer supplied too many descriptors") + if not data: + raise PeerUnavailable("peer closed before complete custody evidence") + content.extend(data) + if len(content) > MAX_MESSAGE_BYTES: + raise PeerUnavailable("peer message exceeds its bound") + line, remainder = bytes(content).split(b"\n", 1) + if remainder: + raise PeerUnavailable("peer supplied an unsolicited additional message") + value = json.loads(line, object_pairs_hook=unique_pairs, parse_constant=_reject_constant) + if not isinstance(value, dict): + raise PeerUnavailable("peer message is not an object") + return value, tuple(received) + except BaseException: + for descriptor in received: + os.close(descriptor) + raise + + +class PeerSession: + """One challenge and independently observed custody, never a launch grant.""" + + def __init__(self, connection: socket.socket, expected: ExpectedPeer, *, timeout: float = 10) -> None: + self.connection = connection + self.expected = expected + self._pidfd = -1 + self._closed = False + self._offered = False + self._held: tuple[HeldFence | HeldResource, ...] = () + self._identities: tuple[ResourceIdentity, ...] = () + self._custody: PeerCustody | None = None + self._timeout = timeout + self.challenge = "" + self.receipt_nonce = secrets.token_hex(32) + try: + if connection.family != socket.AF_UNIX or connection.type not in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET): + raise PeerUnavailable("recovery custody requires an authenticated local socket") + if type(timeout) not in (int, float) or not 0 < timeout <= 60: + raise PeerUnavailable("peer timeout must be positive and at most sixty seconds") + if type(expected.pid) is not int or expected.pid <= 0 or type(expected.uid) is not int or expected.uid < 0: + raise PeerUnavailable("expected peer identity is invalid") + if not expected.start_token.isascii() or not expected.start_token.isdecimal() or str(int(expected.start_token)) != expected.start_token or int(expected.start_token) <= 0: + raise PeerUnavailable("expected peer lifetime is invalid") + if str(uuid.UUID(expected.boot_id)) != expected.boot_id or not all(_name(value) for value in (expected.request_id, expected.authorization_id, expected.target_id)): + raise PeerUnavailable("expected peer binding is invalid") + connection.settimeout(timeout) + self._pidfd = os.pidfd_open(expected.pid, 0) + self._check_lifetime() + message = self._read_message() + if set(message) != {"schema", "request_id", "authorization_id", "target_id", "challenge"} or message.get("schema") != CHALLENGE_SCHEMA: + raise PeerUnavailable("peer challenge schema differs") + if any(message[name] != getattr(expected, name) for name in ("request_id", "authorization_id", "target_id")) or not _token(message["challenge"]): + raise PeerUnavailable("peer challenge binding differs") + self.challenge = str(message["challenge"]) + self._check_lifetime() + except BaseException: + self.close() + raise + + def __enter__(self) -> PeerSession: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def close(self) -> None: + if not self._closed: + self._closed = True + self.connection.close() + if self._pidfd >= 0: + os.close(self._pidfd) + self._pidfd = -1 + + def _check_lifetime(self) -> None: + if self._closed or self._pidfd < 0: + raise PeerUnavailable("peer session is closed") + pid, uid, _ = struct.unpack("3i", self.connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))) + if (pid, uid) != (self.expected.pid, self.expected.uid): + raise PeerUnavailable("local peer credentials differ from the expected target") + poll = select.poll() + poll.register(self._pidfd, select.POLLIN) + prefix, separator, tail = (Path("/proc") / str(pid) / "stat").read_text(encoding="ascii").rpartition(")") + fields = tail.split() + if not separator or prefix.partition(" (")[0] != str(pid) or len(fields) < 20 or fields[0] in ("Z", "X", "x") or fields[19] != self.expected.start_token or Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii").strip() != self.expected.boot_id or poll.poll(0): + raise PeerUnavailable("exact target peer lifetime exited or changed") + + def _set_remaining_timeout(self, deadline: float) -> None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise PeerUnavailable("peer exchange exceeded its bounded deadline") + self.connection.settimeout(remaining) + + def _read_message(self, *, deadline: float | None = None) -> dict[str, object]: + value, _ = receive_peer_message(self.connection, timeout=self._timeout, deadline=deadline) + return value + + def offer_and_verify_custody(self, fences: tuple[HeldFence, ...], resources: tuple[HeldResource, ...] = ()) -> PeerCustody: + """Transfer declared host resources, then require physical receipt proof.""" + try: + if self._offered: + raise PeerUnavailable("this peer session already offered custody") + self._offered = True + self._check_lifetime() + held: tuple[HeldFence | HeldResource, ...] = (*fences, *resources) + if not 3 <= len(held) <= MAX_RESOURCES or not any(fence.kind == "original-sh" for fence in fences) or sum(fence.kind == "target-ex" for fence in fences) != 1 or not any(fence.kind == "transferred-ex" for fence in fences): + raise PeerUnavailable("complete original, target and deployment fence classes are required") + identities = tuple(inspect_held_resource(resource) for resource in held) + if len({item.name for item in identities}) != len(identities) or len({(item.device, item.inode) for item in identities}) != len(identities): + raise PeerUnavailable("resource names or kernel objects are duplicated") + transfer = array.array("i") + offered: list[dict[str, object]] = [] + for resource, identity in zip(held, identities): + transfer_index = None + if identity.kind in ("transferred-ex", "network-namespace"): + transfer_index = len(transfer) + transfer.append(resource.descriptor) + offered.append({"name": identity.name, "kind": identity.kind, "device": identity.device, "inode": identity.inode, "transfer_index": transfer_index}) + binding = {name: getattr(self.expected, name) for name in ("request_id", "authorization_id", "target_id")} + binding.update(challenge=self.challenge, receipt_nonce=self.receipt_nonce) + packet = encode_peer_message({"schema": OFFER_SCHEMA, **binding, "resources": offered}) + deadline = time.monotonic() + self._timeout + self._set_remaining_timeout(deadline) + sent = self.connection.sendmsg([packet], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, transfer)]) + if sent <= 0: + raise PeerUnavailable("custody offer was not sent") + if sent < len(packet): + self._set_remaining_timeout(deadline) + self.connection.sendall(packet[sent:]) + receipt = self._read_message(deadline=deadline) + if set(receipt) != {"schema", *binding, "descriptors"} or receipt.get("schema") != RECEIPT_SCHEMA or any(receipt[name] != value for name, value in binding.items()): + raise PeerUnavailable("custody receipt is stale or bound to another challenge") + descriptors = receipt["descriptors"] + if not isinstance(descriptors, list) or len(descriptors) != len(held): + raise PeerUnavailable("custody receipt omits required descriptors") + peer_descriptors: dict[str, int] = {} + for item in descriptors: + if not isinstance(item, dict) or set(item) != {"name", "fd"} or not _name(item.get("name")) or type(item.get("fd")) is not int: + raise PeerUnavailable("custody receipt descriptor is malformed") + name, descriptor = str(item["name"]), int(item["fd"]) + _fd_path(self.expected.pid, descriptor) + if name in peer_descriptors or descriptor in peer_descriptors.values(): + raise PeerUnavailable("custody receipt duplicates a descriptor") + peer_descriptors[name] = descriptor + if set(peer_descriptors) != {item.name for item in identities}: + raise PeerUnavailable("custody receipt names another resource set") + self._held, self._identities = held, identities + self._check_lifetime() + observed = tuple(_received(identity, self.expected, peer_descriptors[identity.name]) for identity in identities) + result = PeerCustody(self.expected, self.challenge, self.receipt_nonce, observed, time.monotonic_ns()) + self._custody = result + self.check(result) + return result + except BaseException as error: + self.close() + if isinstance(error, (OSError, ValueError, TypeError, KeyError)) and not isinstance(error, PeerUnavailable): + raise PeerUnavailable("current peer custody could not be verified") from error + raise + + def check(self, custody: PeerCustody) -> None: + """Repeat actual custody checks; the caller still owns grant ordering.""" + try: + if self._custody is None or custody != self._custody: + raise PeerUnavailable("custody belongs to another session or was not verified") + self._check_lifetime() + if tuple(inspect_held_resource(resource) for resource in self._held) != self._identities: + raise PeerUnavailable("issuer resource identity or lock changed after receipt") + observed = tuple(_received(item.identity, self.expected, item.peer_descriptor) for item in self._custody.resources) + if observed != self._custody.resources: + raise PeerUnavailable("consumer resource custody changed after receipt") + self._check_lifetime() + except BaseException as error: + self.close() + if isinstance(error, (OSError, ValueError, TypeError, KeyError)) and not isinstance(error, PeerUnavailable): + raise PeerUnavailable("current peer custody could not be verified") from error + raise diff --git a/src/auto_coder/management/recovery_epoch_resources.py b/src/auto_coder/management/recovery_epoch_resources.py new file mode 100644 index 000000000..78b5d83df --- /dev/null +++ b/src/auto_coder/management/recovery_epoch_resources.py @@ -0,0 +1,132 @@ +"""Acquire and retain the actual fences used by one recovery issuer lifetime.""" + +from __future__ import annotations + +import fcntl +import os +import select +import stat +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +from .containment_enrollment import OriginalNetworkPin, _approved_directory +from .original_attempt_evidence import observe as observe_original_history +from .recovery_containment_model import NamedDescriptor, RecoveryEnrollment, RecoveryLock, RecoveryResources +from .recovery_epoch_authorization import EpochAuthorization +from .recovery_epoch_peer import HeldFence, HeldResource, inspect_held_resource +from .recovery_epoch_state import EpochUnavailable, RecoveryEpochState, issuer_lifetime + + +@dataclass(frozen=True) +class EpochResources: + history_descriptor: int + containment: RecoveryResources + fences: tuple[HeldFence, ...] + namespaces: tuple[HeldResource, ...] + + @property + def original_descriptors(self) -> tuple[int, ...]: + return tuple(item.descriptor for item in self.containment.original_owner_fds) + + def check(self) -> None: + if (self.containment.holder_boot_id, self.containment.holder_pid, self.containment.holder_start_token) != issuer_lifetime(): + raise EpochUnavailable("recovery resources escaped their original issuer lifetime") + held: tuple[HeldFence | HeldResource, ...] = (*self.fences, *self.namespaces) + for resource in held: + inspect_held_resource(resource) + + +def _open_bound_lock(selected: RecoveryLock, *, mode: int | None) -> int: + path = Path(selected.path) + _approved_directory(path.parent, uid=os.geteuid()) + if path != path.resolve(strict=True): + raise EpochUnavailable("recovery owner/controller path is not canonical") + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + info, named = os.fstat(descriptor), path.lstat() + expected = (selected.device, selected.inode) + if (info.st_dev, info.st_ino) != expected or (named.st_dev, named.st_ino) != expected or info.st_mode != named.st_mode or not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1 or info.st_uid != os.geteuid(): + raise EpochUnavailable("recovery original fence identity or protection differs") + if mode is not None: + fcntl.flock(descriptor, mode | fcntl.LOCK_NB) + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def _pin_current_namespace(selected: OriginalNetworkPin) -> int: + """Duplicate a currently held nsfs object after pinning its exact holder.""" + boot = issuer_lifetime()[0] + if selected.holder_boot_id != boot: + raise EpochUnavailable("original namespace holder belongs to another boot") + pidfd = os.pidfd_open(selected.holder_pid, 0) + descriptor = -1 + try: + poll = select.poll() + poll.register(pidfd, select.POLLIN) + holder = Path("/proc") / str(selected.holder_pid) + + def check_holder() -> None: + fields = (holder / "stat").read_text(encoding="ascii").rsplit(")", 1)[1].split() + if len(fields) < 20 or fields[0] in ("Z", "X", "x") or fields[19] != selected.holder_start_token or poll.poll(0): + raise EpochUnavailable("original namespace holder exited or changed") + + check_holder() + path = holder / "fd" / str(selected.holder_fd) + before = path.stat() + if os.readlink(path) != selected.namespace or selected.namespace != f"net:[{selected.inode}]" or (before.st_dev, before.st_ino) != (selected.device, selected.inode): + raise EpochUnavailable("original namespace pin no longer names its observed object") + # Following the authenticated process's descriptor is intentional; + # ordinary pathname symlinks never select an original namespace. + descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK) + actual, after = os.fstat(descriptor), path.stat() + if any((item.st_dev, item.st_ino, item.st_mode) != (before.st_dev, before.st_ino, before.st_mode) for item in (actual, after)) or os.readlink(path) != selected.namespace: + raise EpochUnavailable("original namespace object changed during acquisition") + check_holder() + inspect_held_resource(HeldResource("acquired-namespace", descriptor)) + return descriptor + except BaseException: + if descriptor >= 0: + os.close(descriptor) + raise + finally: + os.close(pidfd) + + +@contextmanager +def hold_epoch_resources(authorization: EpochAuthorization, enrollment: RecoveryEnrollment, state: RecoveryEpochState) -> Iterator[EpochResources]: + """Acquire independent original SH and real controller EX; create nothing.""" + with ExitStack() as held: + history = held.enter_context(observe_original_history(authorization.history)) + fences = [HeldFence("history", history, "original-sh")] + owners: list[NamedDescriptor] = [] + controllers: list[NamedDescriptor] = [] + namespaces: list[NamedDescriptor] = [] + transfers: list[HeldResource] = [] + for index, original in enumerate(enrollment.originals): + if original.owner is not None: + owner = _open_bound_lock(original.owner, mode=fcntl.LOCK_SH) + held.callback(os.close, owner) + owners.append(NamedDescriptor(original.generation_id, owner)) + fences.append(HeldFence(f"owner-{index}", owner, "original-sh")) + if original.namespace_pin is not None: + namespace = _pin_current_namespace(original.namespace_pin) + held.callback(os.close, namespace) + namespaces.append(NamedDescriptor(original.generation_id, namespace)) + transfers.append(HeldResource(f"namespace-{index}", namespace)) + for index, watcher in enumerate(enrollment.watchers): + controller = _open_bound_lock(watcher.lock, mode=fcntl.LOCK_EX) + held.callback(os.close, controller) + controllers.append(NamedDescriptor(watcher.unit, controller)) + fences.append(HeldFence(f"controller-{index}", controller, "transferred-ex")) + target = _open_bound_lock(enrollment.target.owner, mode=None) + held.callback(os.close, target) + fences.extend((HeldFence("target-owner", target, "target-ex"), HeldFence("epoch", state.descriptor, "transferred-ex"))) + boot, pid, start = issuer_lifetime() + result = EpochResources(history, RecoveryResources(pid, start, boot, tuple(owners), tuple(controllers), tuple(namespaces)), tuple(fences), tuple(transfers)) + result.check() + yield result + result.check() diff --git a/src/auto_coder/management/recovery_epoch_state.py b/src/auto_coder/management/recovery_epoch_state.py index 7a39317b2..f1adcc146 100644 --- a/src/auto_coder/management/recovery_epoch_state.py +++ b/src/auto_coder/management/recovery_epoch_state.py @@ -52,6 +52,7 @@ class EpochInventory: target_id: str source_sha256: str config_sha256: str + authorization_sha256: str = "" @dataclass(frozen=True) @@ -68,6 +69,8 @@ class EpochBinding: host_boot_id: str host_pid: int host_start_token: str + authorization_sha256: str = "" + evidence_json: str = "" @dataclass(frozen=True) @@ -113,6 +116,8 @@ def _validate_static(value: EpochInventory | EpochBinding) -> None: digest = getattr(value, field) if not isinstance(digest, str) or not _SHA256.fullmatch(digest): raise EpochUnavailable("invalid recovery epoch " + field) + if not isinstance(value.authorization_sha256, str) or (value.authorization_sha256 and not _SHA256.fullmatch(value.authorization_sha256)): + raise EpochUnavailable("invalid recovery epoch authorization digest") def _parse_inventory(encoded: bytes, root: Path, root_id: tuple[int, int], lock_id: tuple[int, int]) -> EpochInventory: @@ -122,8 +127,10 @@ def _parse_inventory(encoded: bytes, root: Path, root_id: tuple[int, int], lock_ raw["enrollment_ids"] = tuple(raw["enrollment_ids"]) inventory = EpochInventory(**raw) _validate_static(inventory) - if type(inventory.schema_version) is not int or inventory.schema_version != 1: + if type(inventory.schema_version) is not int or inventory.schema_version not in (1, 2): raise EpochUnavailable("unsupported recovery epoch inventory schema") + if inventory.schema_version == 2 and not inventory.authorization_sha256: + raise EpochUnavailable("issuer inventory requires its complete authorization digest") if any(type(getattr(inventory, field)) is not int or getattr(inventory, field) < 0 for field in ("root_device", "root_inode", "lock_device", "lock_inode")): raise EpochUnavailable("invalid recovery epoch inventory identity") if inventory.root != str(root) or (inventory.root_device, inventory.root_inode) != root_id or (inventory.lock_device, inventory.lock_inode) != lock_id: @@ -149,6 +156,16 @@ def __init__(self, root: Path, directory: int, lock: int, inventory: EpochInvent self._active = True self._failed = False + @property + def descriptor(self) -> int: + """Return the still-held epoch fence for verified receiver custody.""" + self._check() + return self._lock + + def check(self) -> None: + """Revalidate the original inventory and every durable publication.""" + self._check() + def _check(self) -> None: if not self._active or self._failed or issuer_lifetime() != self._lifetime: raise EpochUnavailable("recovery epoch invocation is closed, failed or inherited") @@ -178,11 +195,17 @@ def _require_binding(self, binding: EpochBinding) -> None: if not isinstance(binding, EpochBinding): raise EpochUnavailable("typed recovery epoch binding required") _validate_static(binding) - for field in ("deployment_id", "authorization_id", "enrollment_ids", "request_id", "preparation_id", "target_id", "source_sha256", "config_sha256"): + for field in ("deployment_id", "authorization_id", "enrollment_ids", "request_id", "preparation_id", "target_id", "source_sha256", "config_sha256", "authorization_sha256"): if getattr(binding, field) != getattr(self.inventory, field): raise EpochUnavailable("recovery epoch binding differs from fixed inventory: " + field) if not isinstance(binding.evidence_sha256, str) or not _SHA256.fullmatch(binding.evidence_sha256): raise EpochUnavailable("invalid independently verified recovery evidence digest") + if not isinstance(binding.evidence_json, str): + raise EpochUnavailable("invalid retained recovery evidence") + if self.inventory.schema_version == 2 or binding.evidence_json: + evidence = binding.evidence_json.encode() + if len(evidence) > _MAX_BYTES // 2 or not isinstance(json.loads(evidence), dict) or canonical(json.loads(evidence)) != evidence or _digest(evidence) != binding.evidence_sha256: + raise EpochUnavailable("retained recovery evidence differs from its complete digest") if type(binding.host_pid) is not int or (binding.host_boot_id, binding.host_pid, binding.host_start_token) != self._lifetime: raise EpochUnavailable("recovery epoch binding differs from current issuer lifetime") if self._binding is not None and binding != self._binding: diff --git a/src/auto_coder/management/recovery_source_profile.py b/src/auto_coder/management/recovery_source_profile.py new file mode 100644 index 000000000..75348dd86 --- /dev/null +++ b/src/auto_coder/management/recovery_source_profile.py @@ -0,0 +1,78 @@ +"""Read the actual complete package source of a selected local Docker object. + +This reader does not execute the package, establish original history, admit a +profile or grant authority. Its caller separately verifies commands and mounts. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import re +import subprocess +import tarfile +from pathlib import PurePosixPath + +_PACKAGE = "/opt/auto-coder/.venv/lib/python3.12/site-packages/auto_coder" +_MAX_ARCHIVE = 64 * 1024 * 1024 +_MAX_SOURCE = 8 * 1024 * 1024 +_MAX_FILES = 4096 + + +def _source_archive_identity(encoded: bytes) -> str: + if not encoded or len(encoded) > _MAX_ARCHIVE: + raise ValueError("recovery source archive is absent or oversized") + sources: dict[PurePosixPath, bytes] = {} + names: set[str] = set() + try: + with tarfile.open(fileobj=io.BytesIO(encoded), mode="r:") as archive: + for index, member in enumerate(archive): + if index >= 2 * _MAX_FILES: + raise ValueError("recovery source archive has too many entries") + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not path.parts or path.parts[0] != "auto_coder" or len(member.name) > 4096 or str(path) != member.name.rstrip("/") or str(path) in names: + raise ValueError("recovery source archive path is ambiguous") + names.add(str(path)) + if member.isdir(): + continue + if not member.isfile() or member.size < 0: + raise ValueError("recovery source archive contains an unsupported entry") + relative = path.relative_to("auto_coder") + if relative.suffix in (".pyc", ".pyo", ".so", ".pyd"): + raise ValueError("recovery source package contains alternate executable import bytes") + if relative.suffix != ".py": + continue + if not 0 <= member.size <= _MAX_SOURCE or len(sources) >= _MAX_FILES: + raise ValueError("recovery source package exceeds its bound") + source = archive.extractfile(member) + if source is None: + raise ValueError("recovery source member is unavailable") + data = source.read(_MAX_SOURCE + 1) + if len(data) != member.size: + raise ValueError("recovery source member is truncated") + sources[relative] = data + except (tarfile.TarError, EOFError, OSError) as error: + raise ValueError("recovery source archive cannot be decoded") from error + if not {PurePosixPath("__init__.py"), PurePosixPath("management/bootstrap_supervisor.py"), PurePosixPath("management/original_attempts.py")} <= set(sources): + raise ValueError("recovery original-history package source is incomplete") + digest = hashlib.sha256() + # Match original_attempts.source_identity's Path ordering and byte framing. + for path in sorted(sources): + name, content = path.as_posix().encode(), sources[path] + digest.update(len(name).to_bytes(8, "big") + name + len(content).to_bytes(8, "big") + content) + return "sha256:" + digest.hexdigest() + + +def read_image_source_identity(container_id: str) -> str: + """Hash current package bytes from this exact object on the local engine.""" + if not isinstance(container_id, str) or not re.fullmatch(r"[0-9a-f]{64}", container_id): + raise ValueError("exact recovery source container ID required") + if os.environ.get("DOCKER_HOST") or os.environ.get("DOCKER_CONTEXT"): + raise ValueError("recovery source endpoint override is unsupported") + context = subprocess.run(("docker", "context", "inspect", "--format", "{{json .Endpoints.docker.Host}}"), capture_output=True, text=True, check=True, timeout=10) + if len(context.stdout) > 4096 or json.loads(context.stdout) != "unix:///var/run/docker.sock": + raise ValueError("recovery source requires the local system Docker engine") + archive = subprocess.run(("docker", "cp", container_id + ":" + _PACKAGE, "-"), capture_output=True, check=True, timeout=30) + return _source_archive_identity(archive.stdout) diff --git a/tests/test_build_provenance.py b/tests/test_build_provenance.py index c77a45f73..9f28f8e07 100644 --- a/tests/test_build_provenance.py +++ b/tests/test_build_provenance.py @@ -31,7 +31,7 @@ class Installed: def _dockerfile_embed_command() -> str: """The production Dockerfile's embed RUN instruction, verbatim.""" lines = (REPO_ROOT / "Dockerfile").read_text(encoding="utf-8").splitlines() - match = [m.group(1) for line in lines if (m := re.match(r"RUN (python -m auto_coder\.build_provenance embed)$", line))] + match = [m.group(1) for line in lines if (m := re.match(r"RUN (python -B -m auto_coder\.build_provenance embed)$", line))] assert len(match) == 1 return match[0] diff --git a/tests/test_management_archived_cutover_container.py b/tests/test_management_archived_cutover_container.py index ffe48ca40..d966db883 100644 --- a/tests/test_management_archived_cutover_container.py +++ b/tests/test_management_archived_cutover_container.py @@ -147,7 +147,11 @@ def path(self, name: str) -> Path: class Chain: - def __init__(self, root: Path, *, with_workers: bool = False, original_history: bool = False): + def __init__(self, root: Path, *, with_workers: bool = False, original_history: bool = False, original_genesis: bool = False): + if original_genesis and original_history: + raise ValueError("original genesis cannot reenroll a prospective history") + self.original_genesis = original_genesis + self.genesis = None self.root = root self.names: list[str] = [] self.generations: list[Generation] = [] @@ -162,7 +166,8 @@ def __init__(self, root: Path, *, with_workers: bool = False, original_history: reservation.bind(("127.0.0.1", 0)) self.port = reservation.getsockname()[1] self.oauth = root / "oauth" - self.oauth.mkdir(mode=0o700) + if not original_genesis: + self.oauth.mkdir(mode=0o700) self.history_binding = None if original_history: history = root / "original-history" @@ -185,6 +190,8 @@ def __init__(self, root: Path, *, with_workers: bool = False, original_history: def prepare_caddy_image(self) -> str: """Use the real pinned Caddy binary with the supported bind-only profile.""" + if self.original_genesis: + return _resolve_fixture_base(os.environ.get("AUTOCODER_RECOVERY_CADDY_IMAGE", "caddy@sha256:504a44d7d74bd2bc28a66afc4774912fc21223ca8a67734af1c54ab2f999b489")) source = _resolve_fixture_base("caddy@sha256:504a44d7d74bd2bc28a66afc4774912fc21223ca8a67734af1c54ab2f999b489") binary = self.root / "caddy" donor = self.docker("create", "--network", "none", source) @@ -249,6 +256,8 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b root = self.root / name root.mkdir(mode=0o700) for part in ("state", "journal", "guard", "bootstrap"): + if self.original_genesis and prior is None and part in ("state", "journal"): + continue (root / part).mkdir(mode=0o700) runtime = { "reference": "bootstrap", @@ -275,7 +284,7 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b if prior is not None: captured = capture_prior_container(prior.container, self.image, expected_mounts=tuple((str(a), b, c) for a, b, c in prior.mounts), require_owner_lock=True, require_deployment_identity=True) self.ensure_proxy(prior) - proxy = capture_prior_container(prior.proxy, self.proxy_image, expected_mounts=((str(prior.path("Caddyfile")), "/etc/caddy/Caddyfile", False),)) + proxy = capture_prior_container(prior.proxy, self.proxy_image, expected_mounts=self.proxy_mounts(prior)) bindings = required_exclusions(captured) config.update(fresh_guard_path="/management-initialization/fresh-guard.sock", fresh_prior_lock_path="/management-prior-owner.lock", archived_exclusions=[] if omit_archive else [asdict(item) for item in bindings]) if self.worker_environment is not None: @@ -329,7 +338,26 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b for item in bindings: mounts += [(Path(item.host_journal_root), item.journal_root, False), (Path(item.host_fence), item.fence, False)] self.configure_profile(name, root, config, runtime, management, mounts, fresh=prior is not None) + if self.original_genesis and prior is None: + config["original_genesis"] = "/bootstrap/original-genesis.json" config_file.write_text(json.dumps(config)) + if self.original_genesis: + management.chmod(0o600) + root_authority = None + if self.original_genesis and prior is None: + from auto_coder.management.original_attempts import canonical + from auto_coder.management.original_genesis import ROOTS_PROFILE, GenesisRootAuthorization, prepare_original_genesis_roots + + caddyfile = self._write_proxy_config(root) + history = self.root / "original-history" + roots_request = GenesisRootAuthorization( + ROOTS_PROFILE, "separately-authorized-new-deployment", "isolated-original-recovery-" + uuid.uuid4().hex, str(uuid.uuid4()), str(config_file), str(management), str(caddyfile), str(root / "state"), str(root / "journal"), str(self.oauth), str(history) + ) + roots_authorization = root / "original-root-authorization.json" + roots_authorization.write_bytes(canonical(asdict(roots_request))) + roots_authorization.chmod(0o600) + root_authority = prepare_original_genesis_roots(roots_authorization) + mounts.append((history, "/management-original-history", False)) container_name = "ancestor-fixture-" + uuid.uuid4().hex[:16] self.names.append(container_name) args = ["create", "--name", container_name, "--read-only", "--publish", f"127.0.0.1:{self.port}:8080"] @@ -338,7 +366,10 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b args += ["--network", "bridge"] for source, target, writable in mounts: args += ["--mount", f"type=bind,src={source},dst={target}" + ("" if writable else ",readonly")] - args += [self.image, "--config", "/bootstrap/config.json"] + if self.original_genesis: + args += ["--env", "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python", self.image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json"] + else: + args += [self.image, "--config", "/bootstrap/config.json"] if prior is not None: args += ["--fresh-initialization"] container = self.docker(*args) @@ -348,6 +379,18 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b assert set(connected) == {"bridge", self.worker_network} generation = Generation(name, root, container, mounts) self.generations.append(generation) + if self.original_genesis: + self._create_recovery_proxy(generation) + if root_authority is not None: + from auto_coder.management.original_genesis import PROFILE, GenesisSelection, capture_original_genesis + + assert generation.proxy is not None + selected = GenesisSelection(PROFILE, roots_request.purpose, roots_request.deployment_id, roots_request.authorization_id, generation.container, generation.proxy, roots_request.history_root, str(config_file), str(management), roots_request.proxy_config_path, str(root_authority)) + authorization = root / "original-capture-authorization.json" + authorization.write_bytes(canonical(asdict(selected))) + authorization.chmod(0o600) + self.genesis = capture_original_genesis(authorization) + self.history_binding = self.genesis.binding if captured is not None: generation.network_fd = os.open(f"/proc/{captured.pid}/ns/net", os.O_RDONLY) self.fds.append(generation.network_fd) @@ -440,6 +483,8 @@ def stop(self, generation: Generation, *, stop_proxy: bool = True) -> None: def ensure_proxy(self, generation: Generation) -> None: if generation.proxy is not None: + if self.original_genesis and self.docker("inspect", "--format", "{{.State.Status}}", generation.proxy) == "created": + self.docker("start", generation.proxy) return actual = json.loads(self.docker("inspect", "--format", "{{json .NetworkSettings.Ports}}", generation.container)) assert actual == {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(self.port)}]}, "actual loopback Caddy port mapping unavailable" @@ -452,6 +497,32 @@ def ensure_proxy(self, generation: Generation) -> None: "run", "-d", "--name", name, "--network", "container:" + generation.container, "--mount", f"type=bind,src={caddyfile},dst=/etc/caddy/Caddyfile,readonly", "--entrypoint", "caddy", self.proxy_image, "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile" ) + def proxy_mounts(self, generation: Generation) -> tuple[tuple[str, str, bool], ...]: + mounts = ((str(generation.path("Caddyfile")), "/etc/caddy/Caddyfile", False),) + if self.original_genesis: + mounts += ((str(generation.path("caddy-data")), "/data", True), (str(generation.path("caddy-config")), "/config", True)) + return mounts + + @staticmethod + def _write_proxy_config(root: Path) -> Path: + path = root / "Caddyfile" + if not path.exists(): + path.write_text("{\n admin off\n auto_https off\n}\nhttp://127.0.0.1:8080 {\n reverse_proxy 127.0.0.1:8081 {\n header_up Host 127.0.0.1:8081\n }\n}\n") + path.chmod(0o600) + return path + + def _create_recovery_proxy(self, generation: Generation) -> None: + """Create the real paired object before original capture, without start.""" + self._write_proxy_config(generation.root) + for name in ("caddy-data", "caddy-config"): + generation.path(name).mkdir(mode=0o700) + name = "original-recovery-proxy-" + uuid.uuid4().hex[:16] + self.names.append(name) + arguments = ["create", "--name", name, "--network", "container:" + generation.container, "--entrypoint", "caddy"] + for source, destination, writable in self.proxy_mounts(generation): + arguments += ["--mount", f"type=bind,src={source},dst={destination}" + ("" if writable else ",readonly")] + generation.proxy = self.docker(*arguments, self.proxy_image, "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile") + def call(self, generation: Generation, tool: str, arguments: dict[str, object], *, authenticated: bool = True): assert generation.proxy is not None request = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool, "arguments": arguments}} diff --git a/tests/test_management_cutover_fixture_contracts.py b/tests/test_management_cutover_fixture_contracts.py index 4920a1764..a6e598a65 100644 --- a/tests/test_management_cutover_fixture_contracts.py +++ b/tests/test_management_cutover_fixture_contracts.py @@ -150,6 +150,7 @@ def test_management_ingress_keeps_workers_on_the_internal_network(tmp_path): fixture.oauth.mkdir() fixture.origin = "http://127.0.0.1:8081" fixture.executable_digest = "f" * 64 + fixture.original_genesis = False fixture.image = "sha256:" + "e" * 64 fixture.port = 23456 fixture.worker_network = "private-workers" diff --git a/tests/test_management_original_genesis.py b/tests/test_management_original_genesis.py new file mode 100644 index 000000000..7bb8f6414 --- /dev/null +++ b/tests/test_management_original_genesis.py @@ -0,0 +1,432 @@ +"""Original genesis filesystem and native-store components. + +Docker observations and container bind mapping are explicit component substitutes. +These tests do not claim real-container, first-child or recovery-epoch acceptance. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import sqlite3 +import uuid +from contextlib import closing +from dataclasses import asdict, replace +from pathlib import Path + +import pytest + +from auto_coder.management import original_genesis as producer +from auto_coder.management.bootstrap_supervisor import BootstrapConfig, ManagementBootstrapSupervisor +from auto_coder.management.containment_observation import DockerObservation +from auto_coder.management.fresh_initialization import claim_fresh_initialization +from auto_coder.management.original_attempt_evidence import derive_lineage, observe, verify_lineage +from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, read_frontier, source_identity +from tests.test_management_original_attempts import _covered, _fingerprint + + +@pytest.fixture +def _use_real_commands(): + pass + + +@pytest.fixture +def _use_real_sleep(): + pass + + +def _write(path, value): + path.write_bytes(canonical(value)) + path.chmod(0o600) + return path + + +@pytest.fixture +def root_authorization(tmp_path, monkeypatch): + # Only the host privilege capability is substituted in this same-UID + # component. Every original file, inode, exclusive create and fsync is real. + monkeypatch.setattr(producer, "_require_host_root", lambda: None) + tmp_path.chmod(0o700) + bootstrap = tmp_path / "bootstrap" + bootstrap.mkdir(mode=0o700) + runtime = {"reference": "bootstrap", "receipt": "sha256:" + "a" * 64, "executable": "/opt/auto-coder/.venv/bin/auto-coder", "arguments": ["management", "serve", "--config", "/bootstrap/management.yaml"]} + _write( + bootstrap / "config.json", + { + "instance_id": "genesis-instance", + "state_root": "/management-child-state", + "journal_root": "/management-operations", + "oauth_root": "/management-oauth", + "listener_url": "http://127.0.0.1:8081/mcp", + "initial_runtime": "bootstrap", + "runtimes": [runtime], + "original_genesis": "/bootstrap/original-genesis.json", + }, + ) + _write( + bootstrap / "management.yaml", + { + "state_root": "/management-oauth", + "operation_state_root": "/management-operations", + "bootstrap_config_path": "/bootstrap/config.json", + "public_origin": "https://management.example.test", + "firebase_project_id": "component-project", + "clients": [{"client_id": "component-client", "redirect_uris": ["https://client.example.test/callback"]}], + "bind_host": "127.0.0.1", + "port": 8081, + }, + ) + proxy = _write(tmp_path / "Caddyfile", {"component": "protected-proxy-config"}) + request = producer.GenesisRootAuthorization( + producer.ROOTS_PROFILE, "separately-authorized-new-deployment", "component-deployment", str(uuid.uuid4()), str(bootstrap / "config.json"), str(bootstrap / "management.yaml"), str(proxy), *(str(tmp_path / role) for role in ("state", "journal", "oauth", "history")) + ) + path = _write(tmp_path / "root-authorization.json", asdict(request)) + return request, path + + +def _capture_component(root_authorization, monkeypatch, tmp_path): + request, authorization = root_authorization + roots_path = producer.prepare_original_genesis_roots(authorization) + selection = producer.GenesisSelection(producer.PROFILE, request.purpose, request.deployment_id, request.authorization_id, "a" * 64, "b" * 64, request.history_root, request.bootstrap_path, request.management_path, request.proxy_config_path, str(roots_path)) + final_authorization = _write(tmp_path / "capture-authorization.json", asdict(selection)) + manager = DockerObservation("component-engine", selection.management_id, "sha256:" + "c" * 64, "d" * 64, "no", 0, False, "created", 0, "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", 0, "e" * 64) + proxy = replace(manager, container_id=selection.proxy_id, image_id="sha256:" + "f" * 64) + mounts = [{"Type": "bind", "Source": str(Path(request.bootstrap_path).parent), "Destination": "/bootstrap", "RW": False}] + mounts += [ + {"Type": "bind", "Source": getattr(request, role + "_root"), "Destination": destination, "RW": role != "history"} + for role, destination in (("state", "/management-child-state"), ("journal", "/management-operations"), ("oauth", "/management-oauth"), ("history", "/management-original-history")) + ] + monkeypatch.setattr(producer, "_created_pair", lambda _selection: ((manager, proxy), {"Mounts": mounts}, source_identity())) + anchor = producer.capture_original_genesis(final_authorization) + return anchor, final_authorization + + +@pytest.fixture +def captured(root_authorization, monkeypatch, tmp_path): + return _capture_component(root_authorization, monkeypatch, tmp_path) + + +def _native_supervisor(anchor, monkeypatch): + """Map the tested container-bind boundary onto real isolated local roots.""" + from auto_coder.management import oauth_config + + mapped = replace(anchor, roots=tuple(replace(root, container_path=root.host_path) for root in anchor.roots)) + original_file = producer._file + original_settings = oauth_config.load_management_settings + original_history = producer.validate_history_genesis + bootstrap = Path(anchor.file("bootstrap").path).parent + + def mapped_file(role, path): + if path.parent == Path("/bootstrap"): + path = bootstrap / path.name + return original_file(role, path) + + def mapped_history(root, *, mounted_genesis=None): + if mounted_genesis is not None: + return mapped + with monkeypatch.context() as current: + current.setattr(producer, "HISTORY_MOUNT", Path("/management-original-history")) + return original_history(root) + + def mapped_settings(path): + settings = original_settings(path) + oauth_root = Path(anchor.root("oauth").host_path) + return replace(settings, state_root=oauth_root, oauth=replace(settings.oauth, database_path=oauth_root / "oauth.sqlite3"), operation_state_root=Path(anchor.root("journal").host_path), bootstrap_config_path=bootstrap / "config.json") + + monkeypatch.setattr(oauth_config, "load_management_settings", mapped_settings) + monkeypatch.setattr(producer, "_file", mapped_file) + monkeypatch.setattr(producer, "HISTORY_MOUNT", Path(anchor.binding.root)) + monkeypatch.setattr(producer, "validate_history_genesis", mapped_history) + config = BootstrapConfig.read(bootstrap / "config.json") + # Map only the executable pathname; the actual receipt and wrapper parser + # still verify this real console file before any native store is created. + launcher = bootstrap / "component-console.py" + launcher.write_text( + "#!/opt/auto-coder/.venv/bin/python\n" + "import sys\n" + "from auto_coder.cli import main\n" + "if __name__ == '__main__':\n" + " if sys.argv[0].endswith('-script.pyw'):\n" + " sys.argv[0] = sys.argv[0][:-11]\n" + " elif sys.argv[0].endswith('.exe'):\n" + " sys.argv[0] = sys.argv[0][:-4]\n" + " sys.exit(main())\n" + ) + runtime = replace(config.runtimes[0], executable=launcher, receipt="sha256:" + hashlib.sha256(launcher.read_bytes()).hexdigest()) + config = replace(config, state_root=Path(anchor.root("state").host_path), journal_root=Path(anchor.root("journal").host_path), oauth_root=Path(anchor.root("oauth").host_path), runtimes=(runtime,)) + return ManagementBootstrapSupervisor(config, bootstrap_config_path=bootstrap / "config.json") + + +def _native_component(anchor, monkeypatch): + supervisor = _native_supervisor(anchor, monkeypatch) + with supervisor._owner_file() as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + result = producer.prepare_original_genesis(supervisor, supervisor._await_owner_start_token()) + assert result is not None + assert producer.prepare_original_genesis(supervisor, supervisor._await_owner_start_token()) is None + # The native creators use WAL. This component explicitly settles its own + # stopped test stores before an immutable reader; the verifier never writes. + for path in producer.native_paths(anchor, host=True).values(): + with closing(sqlite3.connect(path)) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + return supervisor, result + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("state_root", "/wrong-oauth"), + ("operation_state_root", "/wrong-operations"), + ("operation_state_root", None), + ("bootstrap_config_path", "/wrong-bootstrap/config.json"), + ("bootstrap_config_path", None), + ("bind_host", "0.0.0.0"), + ("port", 8082), + ], +) +def test_genesis_capture_refuses_service_using_unrecorded_stores_or_listener(root_authorization, monkeypatch, tmp_path, field, value): + request, _ = root_authorization + management = Path(request.management_path) + settings = json.loads(management.read_bytes()) + settings[field] = value + _write(management, settings) + with pytest.raises(OriginalEvidenceUnavailable, match="effective management"): + _capture_component(root_authorization, monkeypatch, tmp_path) + assert not (management.parent / producer.GENESIS_NAME).exists() + assert all(list(Path(getattr(request, role + "_root")).iterdir()) == [] for role in ("state", "journal", "oauth", "history")) + + +@pytest.mark.parametrize("change", ["receipt", "python-hook", "loader-hook"]) +def test_native_genesis_refuses_unbound_execution_before_original_store_creation(captured, monkeypatch, change): + anchor, _ = captured + supervisor = _native_supervisor(anchor, monkeypatch) + runtime = supervisor.config.runtimes[0] + if change == "receipt": + runtime = replace(runtime, receipt="sha256:" + "f" * 64) + else: + runtime = replace(runtime, environment=(("PYTHONPATH" if change == "python-hook" else "LD_PRELOAD", "/unselected"),)) + supervisor.config = replace(supervisor.config, runtimes=(runtime,)) + with supervisor._owner_file() as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + with pytest.raises(ValueError, match="console receipt differs|environment contains code hooks"): + producer.prepare_original_genesis(supervisor, supervisor._await_owner_start_token()) + assert {path.name for path in supervisor.config.state_root.iterdir()} == {".supervisor-owner.lock"} + assert list(supervisor.config.journal_root.iterdir()) == [] + assert list(supervisor.config.oauth_root.iterdir()) == [] + + +def test_original_roots_are_created_not_inferred_from_emptiness(root_authorization, tmp_path): + request, authorization = root_authorization + Path(request.state_root).mkdir(mode=0o700) + before = _fingerprint(tmp_path) + with pytest.raises(OriginalEvidenceUnavailable, match="no empty-root conversion"): + producer.prepare_original_genesis_roots(authorization) + assert _fingerprint(tmp_path) == before + assert not (Path(request.bootstrap_path).parent / producer.ROOT_CLAIM_NAME).exists() + assert not Path(request.history_root).exists() + + +@pytest.mark.parametrize("boundary", [producer.ROOT_CLAIM_NAME, producer.ROOTS_NAME]) +def test_interrupted_original_root_publication_is_never_reenrolled(root_authorization, tmp_path, monkeypatch, boundary): + request, authorization = root_authorization + original_create = producer._create + + def interrupted(root, name, value): + original_create(root, name, value) + if name == boundary: + raise OSError("interrupted original root publication") + + monkeypatch.setattr(producer, "_create", interrupted) + with pytest.raises(OSError, match="interrupted original"): + producer.prepare_original_genesis_roots(authorization) + monkeypatch.setattr(producer, "_create", original_create) + before = _fingerprint(tmp_path) + with pytest.raises((FileExistsError, OriginalEvidenceUnavailable)): + producer.prepare_original_genesis_roots(authorization) + assert _fingerprint(tmp_path) == before + + +def test_captured_genesis_pins_affirmative_roots_exact_empty_history_and_configs(captured, tmp_path): + anchor, authorization = captured + authority, events = read_frontier(Path(anchor.binding.root), anchor.binding) + assert authority.profile == producer.HISTORY_PROFILE + assert authority.genesis_path == str(Path(anchor.file("bootstrap").path).parent / producer.GENESIS_NAME) + assert events == [] + assert producer.validate_history_genesis(Path(anchor.binding.root)) == anchor + assert {root.role for root in anchor.roots} == {"bootstrap", "state", "journal", "oauth", "history"} + assert {item.role for item in anchor.files} == {"authorization", "root-authorization", "root-claim", "roots", "bootstrap", "management", "proxy"} + before = _fingerprint(tmp_path) + with pytest.raises(OriginalEvidenceUnavailable, match="must be empty|already captured"): + producer.capture_original_genesis(authorization) + assert _fingerprint(tmp_path) == before + for role in ("state", "journal", "oauth"): + assert list(Path(anchor.root(role).host_path).iterdir()) == [] + + +@pytest.mark.parametrize("role", ["bootstrap", "management", "proxy", "root-authorization", "root-claim", "roots", "authorization"]) +def test_genesis_refuses_changed_original_file_without_repair(captured, tmp_path, role): + anchor, _ = captured + original = Path(anchor.file(role).path) + original.write_bytes(original.read_bytes() + b" ") + before = _fingerprint(tmp_path) + with pytest.raises((OriginalEvidenceUnavailable, ValueError)): + read_frontier(Path(anchor.binding.root), anchor.binding) + assert _fingerprint(tmp_path) == before + + +def test_genesis_first_successor_cannot_select_unrelated_baseline(captured, tmp_path): + anchor, _ = captured + intent = _covered(tmp_path, anchor.binding, "foreign-child") + before = _fingerprint(tmp_path) + with pytest.raises(OriginalEvidenceUnavailable, match="uncovered, refused or cyclic"): + claim_fresh_initialization(Path(intent.host_state_root), intent) + assert _fingerprint(tmp_path) == before + assert read_frontier(Path(anchor.binding.root), anchor.binding)[1] == [] + + +def test_native_genesis_unknown_is_complete_evidence_but_missing_store_is_not(captured, tmp_path, monkeypatch): + anchor, _ = captured + _, launch = _native_component(anchor, monkeypatch) + native, first = producer.validate_native_genesis(anchor, Path(anchor.file("bootstrap").path).parent / producer.GENESIS_NAME, host=True) + assert (first["operation_id"], first["attempt_id"]) == launch + assert set(native["stores"]) == {"child", "handoff", "recovery", "operations"} + owner_path = Path(anchor.root("state").host_path) / ".supervisor-owner.lock" + with owner_path.open("rb") as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + with observe(anchor.binding) as history: + evidence = derive_lineage(anchor.binding, history, (owner.fileno(),)) + assert evidence.unavailable == () + assert len(evidence.generations) == 1 + assert evidence.generations[0].outcome == "unknown" + assert verify_lineage(evidence, anchor.binding, history, (owner.fileno(),)) == evidence + before = _fingerprint(tmp_path) + with pytest.raises(OriginalEvidenceUnavailable, match="forged, stale, truncated"): + verify_lineage(replace(evidence, generations=()), anchor.binding, history, (owner.fileno(),)) + assert _fingerprint(tmp_path) == before + database = Path(anchor.root("journal").host_path) / "management-operations.sqlite3" + hidden = database.with_name("removed-original") + database.rename(hidden) + missing = derive_lineage(anchor.binding, history, (owner.fileno(),)) + assert missing.unavailable and any("management-operations.sqlite3" in reason for reason in missing.unavailable) + assert hidden.read_bytes() == before[str(database.relative_to(tmp_path))][2] + + +def test_genesis_full_frontier_retains_original_failed_branch(captured, tmp_path, monkeypatch): + anchor, _ = captured + _native_component(anchor, monkeypatch) + intent = _covered(tmp_path, anchor.binding, "refused-successor") + intent = replace(intent, old_instance=anchor.instance_id, old_container_id=anchor.management_id, oauth_inode=intent.oauth_inode + 1) + with pytest.raises(ValueError, match="retained OAuth root identity differs"): + claim_fresh_initialization(Path(intent.host_state_root), intent) + owner_path = Path(anchor.root("state").host_path) / ".supervisor-owner.lock" + with owner_path.open("rb") as owner: + fcntl.flock(owner.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + with observe(anchor.binding) as history: + evidence = derive_lineage(anchor.binding, history, (owner.fileno(),)) + assert evidence.unavailable == () + assert [(item.instance_id, item.outcome, item.owner_inode > 0) for item in evidence.generations] == [(anchor.instance_id, "unknown", True), (intent.new_instance, "never-created", False)] + assert evidence.attempts[0].request_id == intent.request_id + assert evidence.attempts[0].state == "never-created" + assert verify_lineage(evidence, anchor.binding, history, (owner.fileno(),)) == evidence + + +@pytest.mark.parametrize("missing", [producer.NATIVE_NAME, producer.LAUNCH_NAME]) +def test_interrupted_genesis_native_boundary_cannot_reopen_first_owner(captured, monkeypatch, tmp_path, missing): + anchor, _ = captured + supervisor, _ = _native_component(anchor, monkeypatch) + path = supervisor.config.state_root / missing + path.rename(path.with_suffix(".missing")) + before = _fingerprint(tmp_path) + with pytest.raises(FileExistsError): + with supervisor._owner_file(): + pytest.fail("an incomplete native boundary reopened the original owner") + assert _fingerprint(tmp_path) == before + + +def test_original_genesis_selector_does_not_enable_fresh_or_new(captured): + anchor, _ = captured + config = BootstrapConfig.read(Path(anchor.file("bootstrap").path)) + for changed in (replace(config, new_instance_initialization=True), replace(config, fresh_guard_path=Path("/guard"), fresh_prior_lock_path=Path("/prior"))): + with pytest.raises(ValueError, match="explicitly selected ordinary"): + changed.validate() + assert replace(config, original_genesis=None).original_genesis is None + + +def test_complete_unknown_successor_is_distinct_from_missing_original_claim(captured, tmp_path, monkeypatch): + """Actual original writers/stores; only the host grant transport is substituted.""" + from auto_coder.management import fresh_initialization + from auto_coder.management.bootstrap_store import BootstrapStore + from auto_coder.management.operation_store import ManagementOperationStore + from auto_coder.management.original_attempts import grant_boundary, record_original_stores + from auto_coder.management.recovery_mailbox import RecoveryMailbox + from auto_coder.management.self_update_handoff import HandoffStore + + anchor, _ = captured + _native_component(anchor, monkeypatch) + initial = _covered(tmp_path, anchor.binding, "unknown-successor") + prior_owner_path = Path(anchor.root("state").host_path) / ".supervisor-owner.lock" + prior_info = prior_owner_path.stat() + oauth = anchor.root("oauth") + intent = replace( + initial, + old_instance=anchor.instance_id, + old_container_id=anchor.management_id, + state_root=initial.host_state_root, + journal_root=initial.host_journal_root, + oauth_root=oauth.host_path, + host_oauth_root=oauth.host_path, + oauth_device=oauth.device, + oauth_inode=oauth.inode, + owner_lock_device=prior_info.st_dev, + owner_lock_inode=prior_info.st_ino, + ) + state, journal = Path(intent.host_state_root), Path(intent.host_journal_root) + claim_fresh_initialization(state, intent) + + def transport(_socket, **request): + request.pop("trusted_uid", None) + return {**request, "challenge": "a" * 64} + + monkeypatch.setattr(fresh_initialization, "request_fresh_host_grant", transport) + # This is the production original-admission boundary, not an assertion that + # a real host grant or child was delivered in this component test. + with grant_boundary(intent): + fresh_initialization.request_and_claim_first_launch(state, tmp_path / "unused.sock", owner_incarnation=str(uuid.uuid4())) + owner_path = state / ".supervisor-owner.lock" + owner_path.touch(mode=0o600) + with owner_path.open("rb") as native_owner: + fcntl.flock(native_owner.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + previous_umask = os.umask(0o077) + try: + with closing(BootstrapStore(state, intent.new_instance)._connect()): + pass + with closing(HandoffStore(state, intent.new_instance)._connect()): + pass + with closing(ManagementOperationStore(journal)._connect()): + pass + RecoveryMailbox(state, intent.new_instance).initialize() + record_original_stores(state, intent) + finally: + os.umask(previous_umask) + for database in (*state.glob("*.sqlite3"), *journal.glob("*.sqlite3")): + with closing(sqlite3.connect(database)) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + with prior_owner_path.open("rb") as prior_owner, owner_path.open("rb") as owner: + for stream in (prior_owner, owner): + fcntl.flock(stream.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + fences = (prior_owner.fileno(), owner.fileno()) + with observe(anchor.binding) as history: + evidence = derive_lineage(anchor.binding, history, fences) + assert evidence.unavailable == () + assert [(item.instance_id, item.outcome) for item in evidence.generations] == [(anchor.instance_id, "unknown"), (intent.new_instance, "unknown")] + assert verify_lineage(evidence, anchor.binding, history, fences) == evidence + original = _fingerprint(tmp_path) + claim = state / "fresh-launch-claimed.json" + claim.rename(state / "removed-original-claim") + missing = derive_lineage(anchor.binding, history, fences) + assert missing.unavailable and any("fresh-launch-claimed.json" in reason for reason in missing.unavailable) + assert (state / "removed-original-claim").read_bytes() == original[str(claim.relative_to(tmp_path))][2] + with pytest.raises(OriginalEvidenceUnavailable): + verify_lineage(missing, anchor.binding, history, fences) diff --git a/tests/test_management_original_genesis_capture.py b/tests/test_management_original_genesis_capture.py new file mode 100644 index 000000000..e968d2e85 --- /dev/null +++ b/tests/test_management_original_genesis_capture.py @@ -0,0 +1,72 @@ +"""Bounded ordinary genesis predecessor and stock Caddy capture profiles.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from auto_coder.management import fresh_host_observation as observation +from auto_coder.management.exclusion_host import inherited_support +from auto_coder.management.original_genesis import HISTORY_MOUNT +from tests.test_management_original_genesis import captured, root_authorization # noqa: F401 + + +def test_original_genesis_predecessor_retains_its_exact_readonly_history(captured) -> None: # noqa: F811 + anchor, _ = captured + mounts = [{"Type": "bind", "Source": item.host_path, "Destination": item.container_path, "RW": item.role in {"state", "journal", "oauth"}} for item in anchor.roots] + assert inherited_support(mounts) == ((), {str(HISTORY_MOUNT)}) + mounts[-1]["RW"] = True + with pytest.raises(ValueError): + inherited_support(mounts) + + +def test_original_genesis_predecessor_cannot_replace_only_history_mount(captured, tmp_path: Path) -> None: # noqa: F811 + anchor, _ = captured + alternate = tmp_path / "alternate" + alternate.mkdir(mode=0o700) + mounts = [{"Type": "bind", "Source": item.host_path if item.role != "history" else str(alternate), "Destination": item.container_path, "RW": item.role in {"state", "journal", "oauth"}} for item in anchor.roots] + with pytest.raises(ValueError, match="mounted root differs"): + inherited_support(mounts) + + +@pytest.fixture +def caddy_profile(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + config = tmp_path / "Caddyfile" + config.write_bytes(b"http://127.0.0.1:8080 {}") + paths = [config] + for name in ("data", "config"): + path = tmp_path / name + path.mkdir(mode=0o700) + paths.append(path) + mounts = [{"Type": "bind", "Source": str(path), "Destination": destination, "RW": index > 0} for index, (path, destination) in enumerate(zip(paths, ("/etc/caddy/Caddyfile", "/data", "/config")))] + identifier, image = "a" * 64, "sha256:" + "b" * 64 + values = [identifier, image, True, os.getpid(), "running", False, "", None, None, "container:" + "c" * 64, None, mounts, "unused", "started", 0, ["caddy"], ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]] + marker = object() + monkeypatch.setattr(observation, "_engine_id", lambda: "component-engine") + monkeypatch.setattr(observation, "_inspect", lambda _: values) + monkeypatch.setattr(observation, "_capture_prior_kernel", lambda *_args, **_kwargs: marker) + return identifier, image, values, marker + + +def test_stock_caddy_explicit_private_data_binds_are_captured(caddy_profile) -> None: + identifier, image, values, marker = caddy_profile + expected = tuple((item["Source"], item["Destination"], item["RW"]) for item in values[11]) + assert observation.capture_prior_container(identifier, image, expected_mounts=expected) is marker + + +@pytest.mark.parametrize("mutation", ["partial", "writable-policy", "arbitrary-command", "unprotected-root"]) +def test_caddy_data_profile_cannot_admit_other_mounts(caddy_profile, mutation: str) -> None: + identifier, image, values, _ = caddy_profile + if mutation == "partial": + values[11].pop() + elif mutation == "writable-policy": + values[11][-1]["RW"] = False + elif mutation == "arbitrary-command": + values[15] = ["python"] + else: + Path(values[11][-1]["Source"]).chmod(0o755) + expected = tuple((item["Source"], item["Destination"], item["RW"]) for item in values[11]) + with pytest.raises(observation.HostObservationUnavailable): + observation.capture_prior_container(identifier, image, expected_mounts=expected) diff --git a/tests/test_management_original_genesis_cli.py b/tests/test_management_original_genesis_cli.py new file mode 100644 index 000000000..fbe5f04e1 --- /dev/null +++ b/tests/test_management_original_genesis_cli.py @@ -0,0 +1,36 @@ +"""Actual CLI refusal boundaries, separate from Docker/genesis acceptance.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +def test_selected_genesis_missing_state_root_is_not_recreated_by_diagnostic_logging(tmp_path: Path) -> None: + state = tmp_path / "missing-original-state" + configuration = tmp_path / "config.json" + configuration.write_text( + json.dumps( + { + "instance_id": "original-component", + "state_root": str(state), + "oauth_root": str(tmp_path / "oauth"), + "journal_root": str(tmp_path / "journal"), + "listener_url": "http://127.0.0.1:8081/mcp", + "initial_runtime": "must-not-launch", + "runtimes": [{"reference": "must-not-launch", "receipt": "sha256:" + "a" * 64, "executable": "/opt/auto-coder/.venv/bin/auto-coder", "arguments": ["management", "serve", "--config", "/bootstrap/management.yaml"]}], + "original_genesis": "/bootstrap/original-genesis.json", + } + ) + ) + configuration.chmod(0o600) + result = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.bootstrap_supervisor", "--config", str(configuration)], capture_output=True, text=True, timeout=15) + assert result.returncode != 0 + assert not state.exists(), "diagnostics recreated a missing selected original state root before refusal" + assert not (tmp_path / "journal").exists() and not (tmp_path / "oauth").exists() diff --git a/tests/test_management_recovery_containment.py b/tests/test_management_recovery_containment.py new file mode 100644 index 000000000..b444f6639 --- /dev/null +++ b/tests/test_management_recovery_containment.py @@ -0,0 +1,429 @@ +"""Bounded component tests; native closed-host acceptance is a separate harness.""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import stat +import subprocess +import tarfile +import tempfile +from dataclasses import asdict, replace +from pathlib import Path + +import pytest + +from auto_coder.management import original_attempts +from auto_coder.management import recovery_containment as observer +from auto_coder.management import recovery_source_profile as source +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.containment_route_manifest import DeploymentRootBinding, RecoveryRouteScope, RecoveryRouteSelection, _route_container_ids +from auto_coder.management.recovery_containment_model import RECOVERY_PROFILE, NamedDescriptor, RecoveryContainmentUnavailable, RecoveryEnrollment, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryResources, RecoveryTarget, RecoveryWatcher +from auto_coder.management.recovery_containment_profile import RecoveryManagerFact, _ordinary_mount_effects, assert_disjoint_target + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") +_BOOT = "00000000-0000-4000-8000-000000000001" + + +@pytest.fixture +def _use_real_commands(): + yield + + +@pytest.fixture +def _use_real_sleep(): + yield + + +def _file(path: str, inode: int) -> RecoveryFile: + return RecoveryFile(path, "d" * 64, 1, inode) + + +def _original(index: int = 0, *, started: bool = True) -> RecoveryOriginal: + manager, proxy = format(2 * index + 1, "064x"), format(2 * index + 2, "064x") + base = "/private/generation-" + str(index) + mounts = (DeploymentRootBinding(manager, "/management-child-state", base + "/state", True, 1, 100 + index, stat.S_IFDIR | 0o700), DeploymentRootBinding(proxy, "/etc/caddy/Caddyfile", base + "/Caddyfile", False, 1, 200 + index, stat.S_IFREG | 0o600)) + owner = RecoveryLock(base + "/state/.supervisor-owner.lock", 1, 1000 + index) if started else None + prior = _file(base + "/prior-evidence.json", 2000 + index) if started else None + pin = OriginalNetworkPin(42, "12345", _BOOT, 10 + index, "net:[" + str(3000 + index) + "]", 4, 3000 + index) if started else None + return RecoveryOriginal( + f"generation-{index:02d}", + f"instance-{index:02d}", + manager, + "sha256:" + "a" * 64, + proxy, + "sha256:" + "b" * 64, + "started" if started else "never-started", + _BOOT if started else "", + _file(base + "/bootstrap/config.json", 4000 + index), + _file(base + "/bootstrap/management.yml", 5000 + index), + _file(base + "/Caddyfile", 200 + index), + mounts, + owner, + prior, + pin, + ) + + +def _enrollment(*, generations: int = 1, watchers: int = 1, started: bool = True) -> RecoveryEnrollment: + target_id = "f" * 64 + target = RecoveryTarget( + "target-instance", + target_id, + "sha256:" + "a" * 64, + _file("/private/target/bootstrap/config.json", 9000), + _file("/private/target/bootstrap/management.yml", 9001), + (DeploymentRootBinding(target_id, "/management-child-state", "/private/target/state", True, 1, 9002, stat.S_IFDIR | 0o700),), + RecoveryLock("/private/target/state/.supervisor-owner.lock", 1, 9003), + 18081, + ) + controls = tuple( + RecoveryWatcher(f"auto-coder-management-proxy-watch-{index:02d}.service", _file(f"/private/watcher-{index}/config.json", 10000 + index), _file("/private/package/management/proxy_watch_cli.py", 20000), RecoveryLock(f"/private/watcher-{index}/state/proxy-watch.lock", 1, 30000 + index)) + for index in range(watchers) + ) + return RecoveryEnrollment("approved-management-recovery-containment-v1", "deployment", "enrollment", "authorization", RECOVERY_PROFILE, "local-engine", "sha256:" + "c" * 64, tuple(_original(index, started=started) for index in range(generations)), target, controls, True) + + +def _json(value: object) -> object: + return json.loads(json.dumps(asdict(value))) + + +def test_scope_round_trip_preserves_all_members_and_fixed_profile(): + expected = _enrollment(generations=3, watchers=3) + actual = RecoveryEnrollment.parse(_json(expected)) + assert actual == expected + assert actual.container_ids == tuple(sorted([expected.target.management_id, *(identifier for item in expected.originals for identifier in (item.management_id, item.proxy_id))])) + assert len(actual.container_ids) == 7 + assert actual.approved_observation_only is True + + +@pytest.mark.parametrize("field,value", [("schema", "approved-current-containment-enrollment-v3"), ("profile", "ordinary-v1"), ("approved_observation_only", False), ("source_identity", "same-as-before"), ("deployment_id", "../other"), ("engine_id", "")]) +def test_scope_refuses_foreign_legacy_or_authorizing_schema(field, value): + raw = _json(_enrollment()) + raw[field] = value + with pytest.raises(RecoveryContainmentUnavailable): + RecoveryEnrollment.parse(raw) + + +def test_scope_refuses_unknown_fields_and_container_aliases(): + raw = _json(_enrollment()) + raw["trusted_root"] = "/request/override" + with pytest.raises(RecoveryContainmentUnavailable, match="schema"): + RecoveryEnrollment.parse(raw) + raw.pop("trusted_root") + raw["target"]["management_id"] = raw["originals"][0]["management_id"] + raw["target"]["mounts"][0]["container_id"] = raw["target"]["management_id"] + with pytest.raises(RecoveryContainmentUnavailable, match="alias"): + RecoveryEnrollment.parse(raw) + + +def test_scope_refuses_shared_proxy_identity_between_generations(): + raw = _json(_enrollment(generations=2)) + old_proxy = raw["originals"][0]["proxy_id"] + raw["originals"][1]["proxy_id"] = old_proxy + raw["originals"][1]["mounts"][1]["container_id"] = old_proxy + raw["originals"][1]["mounts"].sort(key=lambda item: (item["container_id"], item["destination"])) + with pytest.raises(RecoveryContainmentUnavailable, match="alias"): + RecoveryEnrollment.parse(raw) + + +def test_never_started_scope_cannot_invent_owner_or_historical_pin(): + selected = _enrollment(started=False) + actual = RecoveryEnrollment.parse(_json(selected)) + assert actual.originals[0].owner is None + assert actual.originals[0].prior_evidence is None + assert actual.originals[0].namespace_pin is None + raw = _json(selected) + raw["originals"][0]["owner"] = asdict(RecoveryLock("/private/fabricated.lock", 1, 2)) + with pytest.raises(RecoveryContainmentUnavailable, match="invent"): + RecoveryEnrollment.parse(raw) + + +def test_scope_requires_original_boot_and_real_namespace_capture_fields(): + raw = _json(_enrollment()) + raw["originals"][0]["boot_id"] = "00000000-0000-4000-8000-000000000002" + with pytest.raises(RecoveryContainmentUnavailable, match="boot"): + RecoveryEnrollment.parse(raw) + raw = _json(_enrollment()) + raw["originals"][0]["namespace_pin"] = None + with pytest.raises(ValueError, match="namespace pin"): + RecoveryEnrollment.parse(raw) + + +@pytest.mark.parametrize("kind", ["owner-path", "owner-inode", "controller-owner", "unsorted"]) +def test_scope_rejects_ambiguous_resource_authority(kind): + raw = _json(_enrollment(generations=2, watchers=2)) + if kind == "owner-path": + raw["originals"][1]["owner"]["path"] = raw["originals"][0]["owner"]["path"] + elif kind == "owner-inode": + raw["originals"][1]["owner"]["inode"] = raw["originals"][0]["owner"]["inode"] + elif kind == "controller-owner": + raw["watchers"][0]["lock"]["inode"] = raw["target"]["owner"]["inode"] + else: + raw["originals"].reverse() + with pytest.raises(RecoveryContainmentUnavailable): + RecoveryEnrollment.parse(raw) + + +def test_complete_peer_resource_bound_is_checked_before_observation(): + boundary = _enrollment(generations=16, watchers=29) + assert RecoveryEnrollment.parse(_json(boundary)) == boundary + with pytest.raises(RecoveryContainmentUnavailable, match="complete peer resource bound"): + RecoveryEnrollment.parse(_json(_enrollment(generations=16, watchers=30))) + + +def test_named_descriptors_are_exact_sorted_and_have_no_aliases(): + expected = RecoveryResources(42, "12345", _BOOT, (NamedDescriptor("generation-00", 10),), (NamedDescriptor("watcher.service", 11),), (NamedDescriptor("generation-00", 12),)) + assert RecoveryResources.parse(_json(expected)) == expected + raw = _json(expected) + raw["namespace_fds"][0]["descriptor"] = 10 + with pytest.raises(RecoveryContainmentUnavailable, match="alias"): + RecoveryResources.parse(raw) + raw = _json(expected) + raw["controller_fds"][0]["descriptor"] = True + with pytest.raises(RecoveryContainmentUnavailable, match="descriptor"): + RecoveryResources.parse(raw) + + +def test_protected_fixture_load_rejects_unknown_fields_duplicates_and_permissions(): + with tempfile.TemporaryDirectory(prefix="recovery-enrollment-component-", dir=Path.home()) as directory: + root = Path(directory) + path = root / "enrollment.json" + selected = _enrollment() + path.write_text(json.dumps(asdict(selected))) + path.chmod(0o600) + actual = observer._load_enrollment("enrollment", root=root, expected_uid=os.geteuid()) + assert actual.enrollment == selected + assert actual.device == path.stat().st_dev and actual.inode == path.stat().st_ino + assert actual.sha256 == hashlib.sha256(path.read_bytes()).hexdigest() + path.chmod(0o644) + with pytest.raises(ValueError, match="protected"): + observer._load_enrollment("enrollment", root=root, expected_uid=os.geteuid()) + path.chmod(0o600) + content = json.dumps(asdict(selected)) + path.write_text(content[:-1] + ',"profile":"ordinary-recovery-v1"}') + with pytest.raises(ValueError, match="duplicate"): + observer._load_enrollment("enrollment", root=root, expected_uid=os.geteuid()) + + +def test_recovery_route_selection_uses_every_member_root_and_configuration(): + selected = _enrollment(generations=3, watchers=3) + approved = observer.RecoveryEnrollmentObservation(selected, "e" * 64, 1, 2, 3) + scope = observer.route_selection(approved) + assert _route_container_ids(scope) == selected.container_ids + assert {item.path for item in scope.enrollment.original_files} >= {item.bootstrap_config.path for item in selected.originals} | {selected.target.bootstrap_config.path} | {item.configuration.path for item in selected.watchers} + forged = RecoveryRouteSelection(RecoveryRouteScope("d", "e", "a", "ordinary-v1", selected.container_ids, ()), "e" * 64) + with pytest.raises(ValueError, match="selection"): + _route_container_ids(forged) + + +def _manager_fact(selected: RecoveryOriginal | RecoveryTarget, *, state: str, journal: str, oauth: str, port: int) -> RecoveryManagerFact: + return RecoveryManagerFact( + selected.instance_id, selected.instance_id, selected.management_id, selected.management_image_id, "component-only", "sha256:" + "c" * 64, "d" * 64, selected.bootstrap_config.sha256, selected.management_config.sha256, state, journal, oauth, selected.owner, port, selected.mounts + ) + + +def test_target_retains_original_oauth_and_separates_state_journal_and_listener(): + selected = _enrollment() + old = _manager_fact(selected.originals[0], state="/private/old/state", journal="/private/old/journal", oauth="/private/oauth", port=18080) + target = _manager_fact(selected.target, state="/private/target/state", journal="/private/target/journal", oauth="/private/oauth", port=18081) + assert_disjoint_target(selected, (old, target)) + with pytest.raises(RecoveryContainmentUnavailable, match="OAuth"): + assert_disjoint_target(selected, (old, replace(target, oauth_root="/private/copied-oauth"))) + with pytest.raises(RecoveryContainmentUnavailable, match="overlaps"): + assert_disjoint_target(selected, (old, replace(target, state_root="/private/old/state/child"))) + with pytest.raises(RecoveryContainmentUnavailable, match="listener"): + assert_disjoint_target(selected, (old, replace(target, published_port=18080))) + + +def test_mount_effects_reject_undeclared_writable_or_interpreter_inputs(): + config = {"state_root": "/management-child-state", "journal_root": "/management-journal", "oauth_root": "/management-oauth", "runtimes": []} + selected = _enrollment().target.mounts[0] + _ordinary_mount_effects(config, (selected,), target=True) + for destination, writable in (("/unselected-data", True), ("/etc/ld.so.preload", False), ("/management-recovery", True)): + with pytest.raises(RecoveryContainmentUnavailable, match="undeclared"): + _ordinary_mount_effects(config, (replace(selected, destination=destination, writable=writable),), target=True) + + +def _archive(entries: list[tuple[str, bytes, bytes | None]]) -> bytes: + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w") as archive: + for name, data, kind in entries: + info = tarfile.TarInfo(name) + info.size = len(data) + if kind is not None: + info.type = kind + info.linkname = "auto_coder/__init__.py" + archive.addfile(info, io.BytesIO(data)) + return output.getvalue() + + +def _sources() -> list[tuple[str, bytes, bytes | None]]: + return [ + ("auto_coder/__init__.py", b"", None), + ("auto_coder/management/bootstrap_supervisor.py", b"# supervisor\n", None), + ("auto_coder/management/original_attempts.py", b"# original\n", None), + ("auto_coder/management.py", b"# flat name\n", None), + ("auto_coder/management/a.py", b"# nested name\n", None), + ] + + +def test_complete_source_archive_matches_existing_source_identity_framing(tmp_path, monkeypatch): + package = tmp_path / "auto_coder" + for name, content, _ in _sources(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + monkeypatch.setattr(original_attempts, "__file__", str(package / "management" / "original_attempts.py")) + assert source._source_archive_identity(_archive(list(reversed(_sources())))) == original_attempts.source_identity() + + +@pytest.mark.parametrize( + "name,kind", + [ + ("auto_coder/__pycache__/__init__.cpython-312.pyc", None), + ("auto_coder/original.pyo", None), + ("auto_coder/native.cpython-312-x86_64-linux-gnu.so", None), + ("auto_coder/native.pyd", None), + ("auto_coder/link.py", tarfile.SYMTYPE), + ("auto_coder/hard.py", tarfile.LNKTYPE), + ("auto_coder/../outside.py", None), + ("/auto_coder/absolute.py", None), + ("auto_coder//ambiguous.py", None), + ("auto_coder/__init__.py", None), + ], +) +def test_source_archive_rejects_alternate_executable_or_ambiguous_members(name, kind): + with pytest.raises(ValueError): + source._source_archive_identity(_archive(_sources() + [(name, b"unsafe", kind)])) + + +def test_source_archive_rejects_missing_history_and_bounded_content(monkeypatch): + with pytest.raises(ValueError, match="incomplete"): + source._source_archive_identity(_archive([("auto_coder/__init__.py", b"", None)])) + monkeypatch.setattr(source, "_MAX_SOURCE", 4) + with pytest.raises(ValueError, match="bound"): + source._source_archive_identity(_archive(_sources())) + + +def test_source_reader_uses_exact_local_object_without_execution(monkeypatch): + calls = [] + archive = _archive(_sources()) + + def command(arguments, **options): + calls.append((arguments, options)) + if arguments[1] == "context": + return subprocess.CompletedProcess(arguments, 0, '"unix:///var/run/docker.sock"', "") + return subprocess.CompletedProcess(arguments, 0, archive, b"") + + monkeypatch.delenv("DOCKER_HOST", raising=False) + monkeypatch.delenv("DOCKER_CONTEXT", raising=False) + monkeypatch.setattr(source.subprocess, "run", command) + assert source.read_image_source_identity("a" * 64) == source._source_archive_identity(archive) + assert calls[1][0] == ("docker", "cp", "a" * 64 + ":/opt/auto-coder/.venv/lib/python3.12/site-packages/auto_coder", "-") + assert not any("exec" in arguments for arguments, _ in calls) + monkeypatch.setenv("DOCKER_HOST", "unix:///unselected.sock") + with pytest.raises(ValueError, match="override"): + source.read_image_source_identity("a" * 64) + assert len(calls) == 2 + + +@pytest.mark.parametrize("mutation", ["grant", "partial", "hash", "unknown"]) +def test_saved_report_cannot_supply_missing_or_authorizing_evidence(monkeypatch, mutation): + raw = { + "schema": "current-recovery-containment-v1", + "status": "observed-current-recovery-containment", + "observation_id": _BOOT, + "observer_pid": 42, + "observer_start_token": "12345", + "boot_id": _BOOT, + "started_monotonic_ns": 1, + "finished_monotonic_ns": 2, + "facts": {}, + "evidence_sha256": observer.digest({}), + "grant_issued": False, + "history_asserted": False, + "admission_created": False, + } + if mutation == "grant": + raw["grant_issued"] = True + elif mutation == "partial": + raw.pop("facts") + elif mutation == "hash": + raw["facts"] = {"forged": True} + else: + raw["repair_history"] = True + monkeypatch.setattr(observer, "observe", lambda *_: pytest.fail("malformed prior report reached current observation")) + with pytest.raises(RecoveryContainmentUnavailable): + observer.verify("enrollment", raw, RecoveryResources(42, "12345", _BOOT, (), (), ())) + + +def _consumer_scope() -> tuple[RecoveryEnrollment, bytes]: + selected = _enrollment() + original, target = selected.originals[0], selected.target + assert original.owner is not None and original.namespace_pin is not None + mounts = ( + *target.mounts, + DeploymentRootBinding(target.management_id, "/management-recovery", "/var/lib/auto-coder/recovery-epoch-peer", False, 1, 50000, stat.S_IFDIR | 0o700), + DeploymentRootBinding(target.management_id, "/management-recovery-history/history.lock", "/private/history/history.lock", False, 1, 50001, stat.S_IFREG | 0o600), + DeploymentRootBinding(target.management_id, "/management-recovery-originals/0/owner.lock", original.owner.path, False, original.owner.device, original.owner.inode, stat.S_IFREG | 0o600), + ) + selected = replace(selected, target=replace(target, mounts=tuple(sorted(mounts, key=lambda item: item.destination)))) + payload = { + "schema_version": 1, + "request_id": "request", + "authorization_id": selected.authorization_id, + "target_id": "target", + "issuer_uid": 0, + "socket_directory_device": 1, + "socket_directory_inode": 50000, + "original_fences": [{"name": "history", "path": "/management-recovery-history/history.lock", "device": 1, "inode": 50001}, {"name": "owner-0", "path": "/management-recovery-originals/0/owner.lock", "device": original.owner.device, "inode": original.owner.inode}], + "target_owner": {"name": "target-owner", "device": target.owner.device, "inode": target.owner.inode}, + "transferred_resources": [ + {"name": "epoch", "kind": "transferred-ex", "device": 1, "inode": 50002}, + {"name": "controller-0", "kind": "transferred-ex", "device": selected.watchers[0].lock.device, "inode": selected.watchers[0].lock.inode}, + {"name": "namespace-0", "kind": "network-namespace", "device": original.namespace_pin.device, "inode": original.namespace_pin.inode}, + ], + } + return selected, json.dumps(payload).encode() + + +def test_target_consumer_inputs_bind_the_exact_socket_owner_history_and_resources(): + selected, encoded = _consumer_scope() + observer._consumer_custody_profile(selected, encoded) + + +@pytest.mark.parametrize("mutation", ["socket-source", "socket-writable", "socket-inode", "owner-source", "owner-inode", "extra-owner", "history-writable", "controller-inode", "namespace-inode", "omitted-epoch"]) +def test_target_consumer_custody_inputs_refuse_substitution_or_omission(mutation): + selected, encoded = _consumer_scope() + payload = json.loads(encoded) + mounts = list(selected.target.mounts) + positions = {item.destination: index for index, item in enumerate(mounts)} + if mutation == "socket-source": + index = positions["/management-recovery"] + mounts[index] = replace(mounts[index], source="/private/other-issuer") + elif mutation == "socket-writable": + index = positions["/management-recovery"] + mounts[index] = replace(mounts[index], writable=True) + elif mutation == "socket-inode": + payload["socket_directory_inode"] += 1 + elif mutation == "owner-source": + index = positions["/management-recovery-originals/0/owner.lock"] + mounts[index] = replace(mounts[index], source="/private/unrelated.lock") + elif mutation == "owner-inode": + payload["original_fences"][1]["inode"] += 1 + elif mutation == "extra-owner": + mounts.append(replace(mounts[positions["/management-recovery-originals/0/owner.lock"]], destination="/management-recovery-originals/1/owner.lock")) + elif mutation == "history-writable": + index = positions["/management-recovery-history/history.lock"] + mounts[index] = replace(mounts[index], writable=True) + elif mutation == "controller-inode": + payload["transferred_resources"][1]["inode"] += 1 + elif mutation == "namespace-inode": + payload["transferred_resources"][2]["inode"] += 1 + else: + payload["transferred_resources"].pop(0) + with pytest.raises(ValueError): + observer._consumer_custody_profile(replace(selected, target=replace(selected.target, mounts=tuple(mounts))), json.dumps(payload).encode()) diff --git a/tests/test_management_recovery_containment_resources.py b/tests/test_management_recovery_containment_resources.py new file mode 100644 index 000000000..1a076465b --- /dev/null +++ b/tests/test_management_recovery_containment_resources.py @@ -0,0 +1,231 @@ +"""Real kernel resource observations, not a complete Docker/host acceptance. + +The typed enrollment is a component selector. Its files and actual holders are +real; no kernel, process, ancestor-protection or flock observer is replaced. +Full production enrollment/profile/controller closure belongs to other tests. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import pwd +import select +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Iterator + +import pytest + +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.recovery_containment_model import NamedDescriptor, RecoveryContainmentUnavailable, RecoveryEnrollment, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryResources, RecoveryTarget, RecoveryWatcher +from auto_coder.management.recovery_containment_resources import hold_observed_resources + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + +_HOLDER = r""" +import fcntl,json,os,sys +from pathlib import Path +paths, inherited = map(json.loads,sys.argv[1:]) +held={} +for name,operation in (('original',fcntl.LOCK_SH),('controller',fcntl.LOCK_EX)): + if name in inherited: + held[name]=inherited[name] + else: + held[name]=os.open(paths[name],os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC) + fcntl.flock(held[name],operation|fcntl.LOCK_NB) +held['namespace']=os.open('/proc/self/ns/net',os.O_RDONLY|os.O_CLOEXEC) +print(json.dumps({'pid':os.getpid(),'start':Path('/proc/self/stat').read_text().rsplit(')',1)[1].split()[19],'boot':Path('/proc/sys/kernel/random/boot_id').read_text().strip(),'descriptors':held}),flush=True) +for command in sys.stdin: + command=command.strip() + if command=='exit': break + if command.startswith('unlock-'): + fcntl.flock(held[command.removeprefix('unlock-')],fcntl.LOCK_UN) + elif command.startswith('close-'): + os.close(held[command.removeprefix('close-')]) + elif command.startswith('replace-'): + name=command.removeprefix('replace-') + previous=held[name] + os.close(previous) + replacement=os.open(paths['other'],os.O_RDONLY|os.O_NOFOLLOW) + if replacement!=previous: + os.dup2(replacement,previous) + os.close(replacement) + else: raise RuntimeError('unsupported holder barrier') + print(command,flush=True) +""" + + +@dataclass +class ResourceCase: + root: Path + process: subprocess.Popen[str] + enrollment: RecoveryEnrollment + resources: RecoveryResources + original_namespace: int + paths: dict[str, Path] + parent_descriptors: list[int] + + def command(self, value: str) -> None: + assert self.process.stdin is not None + self.process.stdin.write(value + "\n") + self.process.stdin.flush() + assert line(self.process) == value + + +def line(process: subprocess.Popen[str]) -> str: + assert process.stdout is not None + assert select.select([process.stdout], [], [], 10)[0], "real resource holder did not reach its barrier" + result = process.stdout.readline().strip() + assert result, (process.poll(), process.stderr.read() if process.poll() is not None and process.stderr is not None else "") + return result + + +def bound(path: Path) -> RecoveryFile: + info = path.stat() + return RecoveryFile(str(path), hashlib.sha256(path.read_bytes()).hexdigest(), info.st_dev, info.st_ino) + + +def selected_lock(path: Path) -> RecoveryLock: + info = path.stat() + return RecoveryLock(str(path), info.st_dev, info.st_ino) + + +@contextmanager +def resource_case(mode: str = "independent") -> Iterator[ResourceCase]: + # Existing containment tests use the real account home to keep every actual + # ancestor protected; pytest's temporary root may live below writable /tmp. + with tempfile.TemporaryDirectory(prefix="ac-recovery-resources-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: + root = Path(directory) + paths = {name: root / name for name in ("original", "controller", "target", "other", "configuration")} + parent_fds: list[int] = [] + process = None + try: + for path in paths.values(): + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + os.close(descriptor) + inherited = {} + if mode.startswith("inherited-"): + name = mode.removeprefix("inherited-") + descriptor = os.open(paths[name], os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + parent_fds.append(descriptor) + fcntl.flock(descriptor, (fcntl.LOCK_SH if name == "original" else fcntl.LOCK_EX) | fcntl.LOCK_NB) + inherited[name] = descriptor + namespace = os.open("/proc/self/ns/net", os.O_RDONLY | os.O_CLOEXEC) + parent_fds.append(namespace) + process = subprocess.Popen([sys.executable, "-I", "-B", "-c", _HOLDER, json.dumps({name: str(path) for name, path in paths.items()}), json.dumps(inherited)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, pass_fds=tuple(inherited.values())) + observed = json.loads(line(process)) + descriptors = observed["descriptors"] + resources = RecoveryResources(observed["pid"], observed["start"], observed["boot"], (NamedDescriptor("generation-one", descriptors["original"]),), (NamedDescriptor("recovery-watch.service", descriptors["controller"]),), (NamedDescriptor("generation-one", descriptors["namespace"]),)) + namespace_info = os.fstat(namespace) + parent_start = Path("/proc/self/stat").read_text().rsplit(")", 1)[1].split()[19] + pin = OriginalNetworkPin(holder_pid=os.getpid(), holder_start_token=parent_start, holder_boot_id=observed["boot"], holder_fd=namespace, namespace=os.readlink(f"/proc/self/fd/{namespace}"), device=namespace_info.st_dev, inode=namespace_info.st_ino) + configuration = bound(paths["configuration"]) + original = RecoveryOriginal("generation-one", "original-one", "a" * 64, "sha256:" + "b" * 64, "c" * 64, "sha256:" + "d" * 64, "started", observed["boot"], configuration, configuration, configuration, (), selected_lock(paths["original"]), configuration, pin) + target = RecoveryTarget("target-one", "e" * 64, "sha256:" + "f" * 64, configuration, configuration, (), selected_lock(paths["target"]), 18080) + watcher = RecoveryWatcher("recovery-watch.service", configuration, configuration, selected_lock(paths["controller"])) + enrollment = RecoveryEnrollment("approved-management-recovery-containment-v1", "deployment-one", "enrollment-one", "authorization-one", "ordinary-recovery-v1", "engine-one", "sha256:" + "0" * 64, (original,), target, (watcher,), True) + yield ResourceCase(root, process, enrollment, resources, namespace, paths, parent_fds) + finally: + if process is not None: + if process.poll() is None: + try: + assert process.stdin is not None + process.stdin.write("exit\n") + process.stdin.flush() + except BrokenPipeError: + pass + try: + process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5) + for descriptor in parent_fds: + try: + os.close(descriptor) + except OSError: + pass + + +def test_real_independent_resource_holder_is_observed_without_observer_flock_authority() -> None: + with resource_case() as case: + before = set(os.listdir("/proc/self/fd")) + with hold_observed_resources(case.enrollment, case.resources, expected_uid=os.geteuid()) as observation: + assert observation.holder_pid == case.process.pid + assert [(item.name, item.kind) for item in observation.resources] == [("generation-one", "original-sh"), ("recovery-watch.service", "controller-ex"), ("generation-one", "network-namespace")] + locks = [item.kernel_lock for item in observation.resources if item.kernel_lock is not None] + assert [(lock.pid, lock.mode) for lock in locks] == [(case.process.pid, "READ"), (case.process.pid, "WRITE")] + contenders = [os.open(case.paths[name], os.O_RDONLY | os.O_NOFOLLOW) for name in ("original", "controller")] + try: + for descriptor in contenders: + with pytest.raises(BlockingIOError): + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + for descriptor in contenders: + os.close(descriptor) + assert set(os.listdir("/proc/self/fd")) == before + # The observer did not acquire an independent lock or release the owner. + case.command("unlock-original") + case.command("unlock-controller") + for name in ("original", "controller"): + descriptor = os.open(case.paths[name], os.O_RDONLY | os.O_NOFOLLOW) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + os.close(descriptor) + + +@pytest.mark.parametrize("mode", ["inherited-original", "inherited-controller"]) +def test_inherited_open_file_description_is_not_independent_issuer_acquisition(mode: str) -> None: + with resource_case(mode) as case: + before = set(os.listdir("/proc/self/fd")) + with pytest.raises(RecoveryContainmentUnavailable, match="independently acquired"): + with hold_observed_resources(case.enrollment, case.resources, expected_uid=os.geteuid()): + pytest.fail("an inherited lock was accepted as independent custody") + assert set(os.listdir("/proc/self/fd")) == before + + +@pytest.mark.parametrize("change", ["unlock-original", "unlock-controller", "close-original", "close-controller", "close-namespace", "replace-original", "replace-controller", "replace-namespace", "holder-death", "original-pin-closed", "owner-path-replaced"]) +def test_real_resource_change_at_observed_barrier_refuses_and_closes_observer_references(change: str) -> None: + with resource_case() as case: + before = len(os.listdir("/proc/self/fd")) + with pytest.raises((ValueError, OSError)): + with hold_observed_resources(case.enrollment, case.resources, expected_uid=os.geteuid()): + if change == "holder-death": + case.process.kill() + case.process.wait(timeout=5) + elif change == "original-pin-closed": + os.close(case.original_namespace) + case.parent_descriptors.remove(case.original_namespace) + elif change == "owner-path-replaced": + case.paths["original"].rename(case.root / "retained-owner") + descriptor = os.open(case.paths["original"], os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + else: + case.command(change) + expected = before - int(change == "original-pin-closed") + assert len(os.listdir("/proc/self/fd")) == expected + + +@pytest.mark.parametrize("field", ["original_owner_fds", "controller_fds", "namespace_fds"]) +def test_required_resource_group_cannot_be_omitted(field: str) -> None: + with resource_case() as case: + changed = replace(case.resources, **{field: ()}) + with pytest.raises(RecoveryContainmentUnavailable, match="membership"): + with hold_observed_resources(case.enrollment, changed, expected_uid=os.geteuid()): + pytest.fail("missing required resource group was accepted") + + +@pytest.mark.parametrize("field,value", [("holder_start_token", "1"), ("holder_boot_id", "00000000-0000-0000-0000-000000000000")]) +def test_holder_lifetime_binding_cannot_name_another_process_instance(field: str, value: str) -> None: + with resource_case() as case: + changed = replace(case.resources, **{field: value}) + with pytest.raises(RecoveryContainmentUnavailable, match="changed or exited"): + with hold_observed_resources(case.enrollment, changed, expected_uid=os.geteuid()): + pytest.fail("another lifetime was accepted") diff --git a/tests/test_management_recovery_epoch_authorization.py b/tests/test_management_recovery_epoch_authorization.py new file mode 100644 index 000000000..d0cb2f696 --- /dev/null +++ b/tests/test_management_recovery_epoch_authorization.py @@ -0,0 +1,212 @@ +"""Fixed authorization/persistence component tests, not host recovery acceptance.""" + +import hashlib +import inspect +import json +import os +import subprocess +import sys +from dataclasses import asdict, replace +from pathlib import Path + +import pytest + +from auto_coder.management.original_attempts import HistoryBinding, canonical +from auto_coder.management.recovery_containment_model import RecoveryFile +from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_SCHEMA, EpochAuthorization, _load_authorization, load_authorization, read_bound_file +from auto_coder.management.recovery_epoch_state import EpochBinding, EpochInventory, EpochUnavailable, _hold_recovery_epoch, issuer_lifetime + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@pytest.fixture +def _use_real_commands(): + pass + + +@pytest.fixture +def _use_real_sleep(): + pass + + +def _write(path, encoded): + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(encoded) + + +def _selected(path): + info = path.stat() + return RecoveryFile(str(path), hashlib.sha256(path.read_bytes()).hexdigest(), info.st_dev, info.st_ino) + + +def _fixture(tmp_path): + authority_root = tmp_path / "authority" + authority_root.mkdir(mode=0o700) + for name in ("consumer.json", "READY.json"): + _write(authority_root / name, b"{}") + authorization = EpochAuthorization( + AUTHORIZATION_SCHEMA, + "deployment-A", + "review-A", + "request-A", + "preparation-A", + "a" * 64, + "enrollment-A", + HistoryBinding(str(tmp_path / "history"), "deployment-A", "00000000-0000-4000-8000-000000000001", "sha256:" + "b" * 64), + _selected(authority_root / "consumer.json"), + _selected(authority_root / "READY.json"), + str(tmp_path / "source"), + str(authority_root / "source-token"), + "/usr/bin/python3", + "/usr/bin/uv", + "0.7.22", + "c" * 40, + authority_root.stat().st_dev, + authority_root.stat().st_ino, + ) + path = authority_root / "authorization.json" + encoded = canonical(asdict(authorization)) + _write(path, encoded) + root = tmp_path / "epoch" + root.mkdir(mode=0o700) + _write(root / "epoch.lock", b"") + lock = (root / "epoch.lock").stat() + inventory = EpochInventory( + 2, + str(root), + root.stat().st_dev, + root.stat().st_ino, + lock.st_dev, + lock.st_ino, + authorization.deployment_id, + authorization.authorization_id, + (authorization.enrollment_id,), + authorization.request_id, + authorization.preparation_id, + authorization.target_id, + "b" * 64, + "d" * 64, + hashlib.sha256(encoded).hexdigest(), + ) + _write(root / "inventory.json", canonical(asdict(inventory))) + return root, path, authorization, inventory + + +def _binding(inventory, *, evidence=None): + evidence = canonical({"originals": [{"journal": "retained", "sha256": "e" * 64}], "outcome": "unknown"}) if evidence is None else evidence + boot, pid, start = issuer_lifetime() + return EpochBinding( + inventory.deployment_id, + inventory.authorization_id, + inventory.enrollment_ids, + inventory.request_id, + inventory.preparation_id, + inventory.target_id, + inventory.source_sha256, + inventory.config_sha256, + hashlib.sha256(evidence).hexdigest(), + boot, + pid, + start, + inventory.authorization_sha256, + evidence.decode(), + ) + + +def test_complete_authorization_is_bound_to_fixed_inventory(tmp_path): + _, path, authorization, inventory = _fixture(tmp_path) + result = _load_authorization(path, inventory) + assert result.authorization == authorization + assert result.original == _selected(path) + assert tuple(inspect.signature(load_authorization).parameters) == ("inventory",) + + +@pytest.mark.parametrize("field,value", [("authorization_id", "other-review"), ("request_id", "other-request"), ("preparation_id", "other-preparation"), ("target_id", "f" * 64), ("deployment_id", "other-deployment"), ("enrollment_ids", ("another-scope",)), ("authorization_sha256", "f" * 64)]) +def test_authorization_cannot_retarget_an_inventory(tmp_path, field, value): + root, path, _, inventory = _fixture(tmp_path) + before = sorted(item.name for item in root.iterdir()) + with pytest.raises(EpochUnavailable): + _load_authorization(path, replace(inventory, **{field: value})) + assert sorted(item.name for item in root.iterdir()) == before + + +def test_legacy_persistence_inventory_does_not_create_issuer_authority(tmp_path): + _, path, _, inventory = _fixture(tmp_path) + with pytest.raises(EpochUnavailable, match="version-two"): + _load_authorization(path, replace(inventory, schema_version=1)) + + +@pytest.mark.parametrize("change", ["partial", "duplicate", "noncanonical", "different-history", "unknown-field", "wrong-target-type"]) +def test_authorization_schema_refuses_ambiguous_selection(tmp_path, change): + _, path, authorization, inventory = _fixture(tmp_path) + raw = asdict(authorization) + if change == "partial": + del raw["ready_receipt"] + elif change == "different-history": + raw["history"]["deployment"] = "elsewhere" + elif change == "unknown-field": + raw["inventory_root"] = "/tmp/another-epoch" + elif change == "wrong-target-type": + raw["target_id"] = True + encoded = canonical(raw) + if change == "duplicate": + encoded = encoded[:-1] + b',"request_id":"request-A"}' + elif change == "noncanonical": + encoded = json.dumps(raw, indent=2).encode() + path.unlink() + _write(path, encoded) + selected = replace(inventory, authorization_sha256=hashlib.sha256(encoded).hexdigest()) + with pytest.raises(ValueError): + _load_authorization(path, selected) + + +def test_version_two_retains_full_canonical_evidence_before_consumption(tmp_path): + root, _, _, inventory = _fixture(tmp_path) + binding = _binding(inventory) + with _hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + state.reserve(binding) + state.record_intent(binding) + receipt = state.consume(binding) + state.check() + assert state.descriptor >= 0 + for phase in ("reserved", "intent", "consumed", "receipt"): + saved = json.loads((root / (phase + ".json")).read_bytes())["binding"] + assert saved["evidence_json"] == binding.evidence_json + assert hashlib.sha256(saved["evidence_json"].encode()).hexdigest() == saved["evidence_sha256"] + assert saved["authorization_sha256"] == inventory.authorization_sha256 + assert receipt.grant_delivered is False + with pytest.raises(EpochUnavailable): + with _hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a consumed version-two epoch was replayed") + + +@pytest.mark.parametrize("evidence", [b"", b"[]", b'{"x": 1}', b'{"x":1,"x":2}', b'{"x":NaN}', b'{"x":"' + b"a" * (512 * 1024) + b'"}']) +def test_version_two_rejects_incomplete_or_ambiguous_evidence_before_reservation(tmp_path, evidence): + root, _, _, inventory = _fixture(tmp_path) + before = {item.name: item.read_bytes() for item in root.iterdir()} + with _hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + with pytest.raises(ValueError): + state.reserve(_binding(inventory, evidence=evidence)) + assert {item.name: item.read_bytes() for item in root.iterdir()} == before + + +def test_retained_evidence_digest_mismatch_cannot_consume(tmp_path): + root, _, _, inventory = _fixture(tmp_path) + binding = _binding(inventory) + with pytest.raises(EpochUnavailable, match="closed, failed or inherited"): + with _hold_recovery_epoch(root, expected_uid=os.geteuid()) as state: + state.reserve(binding) + with pytest.raises(EpochUnavailable, match="digest"): + state.record_intent(replace(binding, evidence_sha256="f" * 64)) + with pytest.raises(EpochUnavailable, match="closed, failed or inherited"): + state.record_intent(binding) + assert not (root / "consumed.json").exists() + + +@pytest.mark.parametrize("argument", ["--root", "--inventory", "--authorization", "--request"]) +def test_host_cli_rejects_request_selected_authority_before_execution(tmp_path, argument): + result = subprocess.run([sys.executable, "-m", "auto_coder.management.recovery_epoch", argument, str(tmp_path)], capture_output=True, text=True, timeout=20) + assert result.returncode == 2 + assert "unrecognized arguments" in result.stderr + assert list(tmp_path.iterdir()) == [] diff --git a/tests/test_management_recovery_epoch_consumer.py b/tests/test_management_recovery_epoch_consumer.py new file mode 100644 index 000000000..ff94b4136 --- /dev/null +++ b/tests/test_management_recovery_epoch_consumer.py @@ -0,0 +1,390 @@ +"""Real consumer custody with a wire issuer, not complete recovery authority. + +The independent issuer owns actual flocks and creates the socket pair, so the +consumer's SO_PEERCRED/SO_PEERPIDFD name that real issuer process. The fixture +supplies protocol grant bytes; it does not claim Docker/lineage/epoch acceptance. +The separate peer tests exercise the issuer's authenticated consumer checks. +""" + +from __future__ import annotations + +import array +import copy +import fcntl +import json +import os +import select +import signal +import socket +import subprocess +import sys +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterator + +import pytest + +from auto_coder.management.recovery_epoch_consumer import ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource, connect_recovery_epoch, parse_consumer_configuration +from auto_coder.management.recovery_epoch_peer import PeerUnavailable + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + +_ISSUER = r""" +import array,fcntl,json,os,secrets,socket,sys +from pathlib import Path +from auto_coder.management.recovery_epoch_peer import OFFER_SCHEMA, RECEIPT_SCHEMA, encode_peer_message, receive_peer_message +from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA +mailbox_fd, paths_json, config_json, mode = sys.argv[1:] +paths, configuration = json.loads(paths_json), json.loads(config_json) +held = {} +for name, operation in (('original', fcntl.LOCK_SH), ('controller', fcntl.LOCK_EX)): + held[name] = os.open(paths[name], os.O_RDONLY|os.O_NOFOLLOW) + fcntl.flock(held[name], operation|fcntl.LOCK_NB) +if any(item['kind']=='network-namespace' for item in configuration['transferred_resources']): + held['namespace'] = os.open('/proc/self/ns/net',os.O_RDONLY|os.O_CLOEXEC) +connection, consumer_endpoint = socket.socketpair(socket.AF_UNIX,socket.SOCK_STREAM) +mailbox=socket.socket(fileno=int(mailbox_fd)) +mailbox.sendmsg([b'c'],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,array.array('i',[consumer_endpoint.fileno()]))]) +consumer_endpoint.close();mailbox.close() +print('issuer-ready',flush=True) +request,_=receive_peer_message(connection,timeout=10) +print('challenge-read',flush=True) +if mode=='cancel-before-offer': + try: + receive_peer_message(connection,timeout=10) + except Exception: + print('consumer-closed-before-offer',flush=True) + sys.stdin.readline() + raise SystemExit(0) + raise RuntimeError('unexpected pre-offer consumer message') +if sys.stdin.readline().strip()!='offer': raise SystemExit(0) +nonce=secrets.token_hex(32) +resources=[] +for item in configuration['original_fences']: + resources.append({'name':item['name'],'kind':'original-sh','device':item['device'],'inode':item['inode'],'transfer_index':None}) +owner=configuration['target_owner'] +resources.append({'name':owner['name'],'kind':'target-ex','device':owner['device'],'inode':owner['inode'],'transfer_index':None}) +transferred=[] +for item in configuration['transferred_resources']: + resources.append({**item,'transfer_index':len(transferred)}) + transferred.append(held[item['name']]) +offer={key:request[key] for key in ('request_id','authorization_id','target_id','challenge')} +offer.update(schema=OFFER_SCHEMA,receipt_nonce=nonce,resources=resources) +if mode=='offer-wrong-nonce': offer['challenge']='0'*64 +if mode=='offer-omitted': resources.pop() +if mode=='offer-extra': resources.append({'name':'undeclared','kind':'transferred-ex','device':1,'inode':2,'transfer_index':0}) +if mode=='offer-wrong-inode': resources[-1]['inode']+=1 +if mode=='offer-original-transfer': resources[0]['transfer_index']=0 +if mode=='offer-extra-fd': transferred.append(held['original']) +if mode=='offer-unheld-ex': fcntl.flock(held['controller'],fcntl.LOCK_UN) +connection.sendmsg([encode_peer_message(offer)],[(socket.SOL_SOCKET,socket.SCM_RIGHTS,array.array('i',transferred))]) +try: + receipt,_=receive_peer_message(connection,timeout=10) +except Exception: + print('receipt-unavailable',flush=True) + sys.stdin.readline() + raise SystemExit(0) +assert receipt['schema']==RECEIPT_SCHEMA and receipt['receipt_nonce']==nonce +assert {item['name'] for item in receipt['descriptors']}=={item['name'] for item in resources} +print('receipt-read',flush=True) +if sys.stdin.readline().strip()!='grant': raise SystemExit(0) +grant={key:request[key] for key in ('request_id','authorization_id','target_id','challenge')} +grant.update(schema=GRANT_SCHEMA,granted='recovery-epoch-consumed',receipt_nonce=nonce,inventory_sha256='1'*64,binding_sha256='2'*64,consumed_sha256='3'*64,receipt_sha256='4'*64,host_boot_id=Path('/proc/sys/kernel/random/boot_id').read_text().strip(),host_pid=os.getpid(),host_start_token=Path('/proc/self/stat').read_text().rsplit(')',1)[1].split()[19]) +if mode=='grant-wrong-target': grant['target_id']='another-target' +if mode=='grant-stale-nonce': grant['receipt_nonce']='0'*64 +if mode=='grant-missing-consumption': grant['consumed_sha256']='' +if mode=='grant-wrong-host': grant['host_pid']+=1 +if mode=='grant-extra-field': grant['another_authority']=True +connection.sendall(encode_peer_message(grant)) +print('grant-sent',flush=True) +for command in sys.stdin: + command=command.strip() + if command=='exit': break + if command=='close-channel': connection.close() + elif command=='unlock-controller': fcntl.flock(held['controller'],fcntl.LOCK_UN) + else: raise RuntimeError('unsupported issuer command') + print(command,flush=True) +""" + +_CONSUMER = r""" +import fcntl,json,os,socket,sys +from dataclasses import asdict +from pathlib import Path +from auto_coder.management.recovery_epoch_consumer import RecoveryEpochConsumer,parse_consumer_configuration +fd,config_path,target_path=sys.argv[1:] +connection=socket.socket(fileno=int(fd)) +configuration=parse_consumer_configuration(Path(config_path).read_bytes()) +target=os.open(target_path,os.O_RDONLY|os.O_NOFOLLOW) +fcntl.flock(target,fcntl.LOCK_EX|fcntl.LOCK_NB) +try: + handle=RecoveryEpochConsumer(connection,configuration,target,timeout=10,configuration_file=Path(config_path)) +except Exception as error: + print(json.dumps({'status':'refused','error':type(error).__name__,'detail':str(error)}),flush=True) + sys.stdin.readline() + raise SystemExit(0) +print(json.dumps({'status':'received','grant':asdict(handle.grant),'retained':len(handle.retained_descriptors)}),flush=True) +for command in sys.stdin: + command=command.strip() + if command=='exit': handle.close();break + if command=='close': + handle.close();print(json.dumps({'status':'closed'}),flush=True);continue + if command!='check': raise RuntimeError('unsupported consumer command') + try: + handle.check_current() + result={'status':'current'} + except Exception as error: + result={'status':'unavailable','error':type(error).__name__,'retained':len(handle.retained_descriptors),'all_open':all(os.fstat(fd).st_ino>0 for fd in handle.retained_descriptors)} + print(json.dumps(result),flush=True) +""" + + +_BOOTSTRAP = r""" +import json,os,socket,sys +from pathlib import Path +from auto_coder.management import recovery_epoch_bootstrap as bootstrap +from auto_coder.management.bootstrap_supervisor import BootstrapConfig,RuntimeSpec +from auto_coder.management.recovery_epoch_consumer import CONSUMER_CONFIG,RecoveryEpochConsumer,parse_consumer_configuration +fd,config_path,target_path=sys.argv[1:] +connection=socket.socket(fileno=int(fd)) +configuration=parse_consumer_configuration(Path(config_path).read_bytes()) +state=Path(target_path).parent +root=state.parent +for name in ('journal','oauth'): + (root/name).mkdir(mode=0o700) +config=BootstrapConfig('target-one',state,root/'oauth',root/'journal','http://127.0.0.1:8080','never-launched',(RuntimeSpec('never-launched','sha256:'+'1'*64,Path(sys.executable),('-c','raise RuntimeError("must not launch")')),),verification_timeout=10,recovery_epoch_config=CONSUMER_CONFIG) +# Component seam: use the real connected wire fixture, preserving the actual +# bootstrap ownership acquisition, signal handlers, consumer, and cleanup. +def connected_fixture(config_path,owner,*,timeout): + assert config_path==CONSUMER_CONFIG + return RecoveryEpochConsumer(connection,configuration,owner,timeout=timeout) +bootstrap.connect_recovery_epoch=connected_fixture +status=bootstrap.run_recovery_epoch_bootstrap(config) +print(json.dumps({'bootstrap_status':status}),flush=True) +raise SystemExit(status) +""" + + +@dataclass +class Exchange: + issuer: subprocess.Popen[str] + consumer: subprocess.Popen[str] + configuration: ConsumerConfiguration + configuration_path: Path + paths: dict[str, str] + + +def line(process: subprocess.Popen[str]) -> str: + assert process.stdout is not None + assert select.select([process.stdout], [], [], 15)[0], "independent process did not reach the required barrier" + value = process.stdout.readline().strip() + assert value, (process.poll(), process.stderr.read() if process.poll() is not None and process.stderr is not None else "") + return value + + +def command(process: subprocess.Popen[str], value: str) -> None: + assert process.stdin is not None + process.stdin.write(value + "\n") + process.stdin.flush() + + +def configuration(root: Path, *, namespace: bool = False, bootstrap: bool = False) -> tuple[ConsumerConfiguration, dict[str, str]]: + paths = {name: str(root / name) for name in ("original", "target", "controller")} + if bootstrap: + state = root / "state" + state.mkdir(mode=0o700) + paths["target"] = str(state / ".supervisor-owner.lock") + for path in paths.values(): + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + original_info, target_info, controller_info = (Path(paths[name]).stat() for name in ("original", "target", "controller")) + directory = root.stat() + transferred = [TransferredResource("controller", "transferred-ex", controller_info.st_dev, controller_info.st_ino)] + if namespace: + info = Path("/proc/self/ns/net").stat() + transferred.append(TransferredResource("namespace", "network-namespace", info.st_dev, info.st_ino)) + return ( + ConsumerConfiguration( + 1, "request-one", "authorization-one", "target-one", os.geteuid(), directory.st_dev, directory.st_ino, (OriginalFence("original", paths["original"], original_info.st_dev, original_info.st_ino),), TargetOwner("target", target_info.st_dev, target_info.st_ino), tuple(transferred) + ), + paths, + ) + + +@contextmanager +def exchange(tmp_path: Path, mode: str = "valid", *, namespace: bool = False, bootstrap: bool = False) -> Iterator[Exchange]: + selected, paths = configuration(tmp_path, namespace=namespace, bootstrap=bootstrap) + config_path = tmp_path / "configuration.json" + config_path.write_text(json.dumps(asdict(selected))) + config_path.chmod(0o600) + mailbox, issuer_mailbox = socket.socketpair() + mailbox.settimeout(10) + issuer = consumer = None + endpoint = None + try: + issuer = subprocess.Popen([sys.executable, "-B", "-c", _ISSUER, str(issuer_mailbox.fileno()), json.dumps(paths), json.dumps(asdict(selected)), mode], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, pass_fds=(issuer_mailbox.fileno(),)) + issuer_mailbox.close() + payload, ancillary, flags, _ = mailbox.recvmsg(1, socket.CMSG_SPACE(array.array("i").itemsize), socket.MSG_CMSG_CLOEXEC) + assert payload == b"c" and not flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC) and len(ancillary) == 1 + descriptors = array.array("i") + descriptors.frombytes(ancillary[0][2]) + assert len(descriptors) == 1 + endpoint = descriptors[0] + assert line(issuer) == "issuer-ready" + consumer = subprocess.Popen([sys.executable, "-B", "-c", _BOOTSTRAP if bootstrap else _CONSUMER, str(endpoint), str(config_path), paths["target"]], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, pass_fds=(endpoint,)) + os.close(endpoint) + endpoint = None + assert line(issuer) == "challenge-read" + yield Exchange(issuer, consumer, selected, config_path, paths) + finally: + if endpoint is not None: + os.close(endpoint) + mailbox.close() + issuer_mailbox.close() + for process in (consumer, issuer): + if process is None: + continue + if process.poll() is None: + try: + command(process, "exit") + except BrokenPipeError: + pass + try: + process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5) + + +def grant(case: Exchange) -> dict[str, object]: + command(case.issuer, "offer") + assert line(case.issuer) == "receipt-read" + assert case.consumer.poll() is None + command(case.issuer, "grant") + assert line(case.issuer) == "grant-sent" + return json.loads(line(case.consumer)) + + +@pytest.mark.parametrize("namespace", [False, True]) +def test_real_consumer_retains_original_target_controller_and_namespace_custody(tmp_path: Path, namespace: bool) -> None: + with exchange(tmp_path, namespace=namespace) as case: + result = grant(case) + assert result["status"] == "received" and result["retained"] == 2 + int(namespace) + assert result["grant"]["host_pid"] == case.issuer.pid + command(case.consumer, "check") + assert json.loads(line(case.consumer)) == {"status": "current"} + assert sorted(path.name for path in tmp_path.iterdir()) == ["configuration.json", "controller", "original", "target"] + + +@pytest.mark.parametrize("mode", ["offer-wrong-nonce", "offer-omitted", "offer-extra", "offer-wrong-inode", "offer-original-transfer", "offer-extra-fd", "offer-unheld-ex"]) +def test_consumer_refuses_incomplete_or_wrong_physical_offer(tmp_path: Path, mode: str) -> None: + with exchange(tmp_path, mode) as case: + command(case.issuer, "offer") + result = json.loads(line(case.consumer)) + assert result["status"] == "refused" and result["error"] == "PeerUnavailable" + assert line(case.issuer) == "receipt-unavailable" + + +@pytest.mark.parametrize("mode", ["grant-wrong-target", "grant-stale-nonce", "grant-missing-consumption", "grant-wrong-host", "grant-extra-field"]) +def test_consumer_refuses_wrong_or_unbound_grant(tmp_path: Path, mode: str) -> None: + with exchange(tmp_path, mode) as case: + result = grant(case) + assert result["status"] == "refused" and result["error"] == "PeerUnavailable" + + +@pytest.mark.parametrize("failure", ["host-death", "channel-closed", "controller-unlocked", "config-changed", "config-replaced"]) +def test_current_failure_preserves_received_custody_until_explicit_close(tmp_path: Path, failure: str) -> None: + with exchange(tmp_path, namespace=True) as case: + assert grant(case)["status"] == "received" + if failure == "host-death": + case.issuer.kill() + case.issuer.wait(timeout=5) + elif failure == "channel-closed": + command(case.issuer, "close-channel") + assert line(case.issuer) == "close-channel" + elif failure == "controller-unlocked": + command(case.issuer, "unlock-controller") + assert line(case.issuer) == "unlock-controller" + elif failure == "config-changed": + with case.configuration_path.open("ab") as stream: + stream.write(b"\n") + else: + encoded = case.configuration_path.read_bytes() + case.configuration_path.rename(tmp_path / "preserved-configuration.json") + case.configuration_path.write_bytes(encoded) + case.configuration_path.chmod(0o600) + command(case.consumer, "check") + result = json.loads(line(case.consumer)) + assert result == {"status": "unavailable", "error": "PeerUnavailable", "retained": 3, "all_open": True} + command(case.consumer, "close") + assert json.loads(line(case.consumer)) == {"status": "closed"} + + +def test_issuer_death_keeps_original_and_transferred_fences_exclusive_until_consumer_closes(tmp_path: Path) -> None: + with exchange(tmp_path) as case: + assert grant(case)["status"] == "received" + case.issuer.kill() + case.issuer.wait(timeout=5) + command(case.consumer, "check") + assert json.loads(line(case.consumer))["status"] == "unavailable" + contenders = [os.open(case.paths[name], os.O_RDONLY | os.O_NOFOLLOW) for name in ("original", "controller")] + try: + for descriptor in contenders: + with pytest.raises(BlockingIOError): + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + command(case.consumer, "close") + assert json.loads(line(case.consumer)) == {"status": "closed"} + for descriptor in contenders: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + for descriptor in contenders: + os.close(descriptor) + + +@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGINT]) +def test_bootstrap_cancellation_before_offer_closes_connection_without_waiting_for_grant(tmp_path: Path, signum: int) -> None: + with exchange(tmp_path, "cancel-before-offer", bootstrap=True) as case: + # The observed challenge proves that the actual bootstrap signal handler + # is installed and its consumer is blocked before any physical offer. + assert case.consumer.poll() is None + case.consumer.send_signal(signum) + assert case.consumer.wait(timeout=5) == 1 + assert line(case.issuer) == "consumer-closed-before-offer" + stdout, stderr = case.consumer.communicate(timeout=5) + assert '{"bootstrap_status": 1}' in stdout, (stdout, stderr) + assert "Recovery custody acknowledged" not in stdout + assert not any((tmp_path / "journal").iterdir()) + assert sorted(path.name for path in (tmp_path / "state").iterdir()) == [".supervisor-owner.lock", "logs"] + descriptor = os.open(case.paths["target"], os.O_RDONLY | os.O_NOFOLLOW) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + os.close(descriptor) + + +@pytest.mark.parametrize("change", ["duplicate-name", "duplicate-object", "duplicate-path", "missing-controller", "unknown-kind", "boolean-identity", "extra-field"]) +def test_consumer_configuration_is_complete_and_unambiguous(tmp_path: Path, change: str) -> None: + selected, _ = configuration(tmp_path) + raw = copy.deepcopy(asdict(selected)) + assert parse_consumer_configuration(json.dumps(raw).encode()) == selected + if change == "duplicate-name": + raw["target_owner"]["name"] = "original" + elif change == "duplicate-object": + raw["target_owner"].update(device=raw["original_fences"][0]["device"], inode=raw["original_fences"][0]["inode"]) + elif change == "duplicate-path": + raw["original_fences"] = (*raw["original_fences"], {**raw["original_fences"][0], "name": "another", "inode": 1234}) + elif change == "missing-controller": + raw["transferred_resources"] = [] + elif change == "unknown-kind": + raw["transferred_resources"][0]["kind"] = "unverified-ex" + elif change == "boolean-identity": + raw["target_owner"]["inode"] = True + else: + raw["root_override"] = str(tmp_path) + with pytest.raises(PeerUnavailable): + parse_consumer_configuration(json.dumps(raw).encode()) + + +def test_production_factory_rejects_config_path_override_before_connecting(tmp_path: Path) -> None: + with pytest.raises(PeerUnavailable, match="fixed bootstrap path"): + connect_recovery_epoch(tmp_path / "configuration.json", -1) diff --git a/tests/test_management_recovery_epoch_fault_tracer.py b/tests/test_management_recovery_epoch_fault_tracer.py new file mode 100644 index 000000000..8d51f5e32 --- /dev/null +++ b/tests/test_management_recovery_epoch_fault_tracer.py @@ -0,0 +1,135 @@ +"""Native syscall barrier checks, separate from full VM issuer acceptance. + +These tests trace the real durable-state component in independent processes. +The VM workflow separately traces the actual zero-argument host issuer with +real Docker/history/containment/READY and authenticated consumer custody. +""" + +import errno +import os +import signal +import subprocess +import sys +from pathlib import Path + +import pytest +from scripts.recovery_epoch_fault_tracer import BOUNDARY_SCENARIOS, PUBLICATIONS, KernelPublicationBarrier, parse_boundary + +from auto_coder.management.recovery_epoch_state import EpochUnavailable, _hold_recovery_epoch +from tests.test_management_recovery_epoch_state import _inventory, _snapshot + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@pytest.fixture +def _use_real_commands(): + pass + + +@pytest.fixture +def _use_real_sleep(): + pass + + +_CHILD = """ +import os +import sys +from pathlib import Path +from auto_coder.management.recovery_epoch_state import EpochBinding, _hold_recovery_epoch, issuer_lifetime +assert os.read(int(sys.argv[2]), 1) == b'1' +os.close(int(sys.argv[2])) +with _hold_recovery_epoch(Path(sys.argv[1]), expected_uid=os.geteuid()) as state: + inventory = state.inventory + boot, pid, start = issuer_lifetime() + binding = EpochBinding(inventory.deployment_id, inventory.authorization_id, inventory.enrollment_ids, inventory.request_id, inventory.preparation_id, inventory.target_id, inventory.source_sha256, inventory.config_sha256, 'd' * 64, boot, pid, start) + state.reserve(binding) + state.record_intent(binding) + state.consume(binding) +print('all-publications-completed', flush=True) +""" + + +def _start(process: subprocess.Popen, root: Path, name: str, when: str) -> KernelPublicationBarrier: + barrier = KernelPublicationBarrier(process, root=root, name=name, when=when) + try: + barrier.start() + except OSError as error: + barrier.close() + if error.errno in (errno.EPERM, errno.EACCES): + pytest.skip("this environment prohibits actual ptrace; full native VM acceptance remains required") + raise + return barrier + + +@pytest.mark.parametrize("scenario", BOUNDARY_SCENARIOS) +def test_real_syscall_stop_preserves_exact_publication_prefix(tmp_path: Path, scenario: str) -> None: + root, _ = _inventory(tmp_path) + name, when = parse_boundary(scenario) + reader, writer = os.pipe() + process = subprocess.Popen([sys.executable, "-B", "-c", _CHILD, str(root), str(reader)], pass_fds=(reader,), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + os.close(reader) + barrier = None + try: + barrier = _start(process, root, name, when) + os.write(writer, b"1") + barrier.resume_initial() + observation = barrier.wait(timeout=20) + assert observation.process_id == process.pid and observation.name == name and observation.when == when + count = PUBLICATIONS.index(name) + (when == "after") + assert {path.name for path in root.iterdir()} == {"epoch.lock", "inventory.json", *PUBLICATIONS[:count]} + assert observation.file_fsync_observed == (when == "after") + assert observation.directory_fsync_observed == (when == "after") + assert (observation.file_sha256 is not None) == (when == "after") + retained = _snapshot(root) + barrier.kill() + assert process.returncode == -signal.SIGKILL + assert _snapshot(root) == retained + if count: + with pytest.raises(EpochUnavailable, match="existing, partial or conflicting"): + with _hold_recovery_epoch(root, expected_uid=os.geteuid()): + pytest.fail("a published original record was reused") + assert _snapshot(root) == retained + finally: + os.close(writer) + if barrier is not None: + barrier.close() + if process.poll() is None: + process.kill() + process.communicate(timeout=15) + + +def test_releasing_actual_syscall_barrier_continues_original_production_code(tmp_path: Path) -> None: + root, _ = _inventory(tmp_path) + reader, writer = os.pipe() + process = subprocess.Popen([sys.executable, "-B", "-c", _CHILD, str(root), str(reader)], pass_fds=(reader,), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + os.close(reader) + barrier = None + try: + barrier = _start(process, root, "intent-head.json", "after") + os.write(writer, b"1") + barrier.resume_initial() + barrier.wait(timeout=20) + assert not (root / "consumed.json").exists() + barrier.release() + stdout, stderr = process.communicate(timeout=20) + assert process.returncode == 0, stderr + assert "all-publications-completed" in stdout + assert {path.name for path in root.iterdir()} == {"epoch.lock", "inventory.json", *PUBLICATIONS} + finally: + os.close(writer) + if barrier is not None: + barrier.close() + if process.poll() is None: + process.kill() + process.communicate(timeout=15) + + +@pytest.mark.parametrize("scenario", ["", "before", "after-inventory", "before-../reserved", "before-reserved.json", "before-reserved-head-extra", "after-consumed-root"]) +def test_boundary_selector_cannot_choose_unreviewed_paths(scenario: str) -> None: + with pytest.raises(ValueError, match="reviewed publication"): + parse_boundary(scenario) + + +def test_all_reviewed_record_and_head_boundaries_are_explicit() -> None: + assert len(BOUNDARY_SCENARIOS) == 16 and len(set(BOUNDARY_SCENARIOS)) == 16 + assert {parse_boundary(item) for item in BOUNDARY_SCENARIOS} == {(name, when) for name in PUBLICATIONS for when in ("before", "after")} diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py new file mode 100644 index 000000000..b34318119 --- /dev/null +++ b/tests/test_management_recovery_epoch_host.py @@ -0,0 +1,223 @@ +"""Authorization gates and acceptance oracles for the real recovery VM driver. + +These are component tests of the driver, never substitutes for its native VM +run. The separate workflow runs the driver against genuine Docker/systemd, +original stores, qualified source publication, READY and peer custody. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +from pathlib import Path + +import pytest +from scripts import test_management_recovery_epoch_host as fixture + +from auto_coder.management.original_attempts import canonical +from tests.test_management_archived_cutover_container import Generation + + +def _inputs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: + source = tmp_path / "source" + source.mkdir() + token = tmp_path / "token" + token.write_text("component-token-placeholder") + for name in ("AUTOCODER_FRESH_TEST_BASE_IMAGE", "AUTOCODER_RECOVERY_CADDY_IMAGE"): + monkeypatch.setenv(name, "sha256:" + "a" * 64) + return {"authorization_id": "test-authorization", "source_repository": source, "source_token_file": token, "source_sha": "b" * 40, "uv": Path(sys.executable).resolve(), "uv_version": "0.13.6", "output": tmp_path / "result.json"} + + +def test_invalid_source_stops_before_host_authorization(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + inputs["source_sha"] = "main" + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: pytest.fail("host authorization must not be reached")) + with pytest.raises(ValueError, match="exact qualified source"): + fixture.run_fixture(**inputs) + assert not inputs["output"].exists() + + +def test_unmatched_marker_stops_before_fixture_provisioning(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + created = [] + + def refusal(_): + raise PermissionError("the actual original marker does not select this host") + + monkeypatch.setattr(fixture, "require_disposable_host", refusal) + monkeypatch.setattr(fixture, "_private_directory", lambda path: created.append(path)) + monkeypatch.setattr(fixture.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("unauthorized driver attempted a host effect")) + with pytest.raises(PermissionError, match="original marker"): + fixture.run_fixture(**inputs) + assert created == [] and not inputs["output"].exists() + + +def test_mutable_image_tag_stops_before_host_authorization(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + monkeypatch.setenv("AUTOCODER_FRESH_TEST_BASE_IMAGE", "auto-coder:latest") + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: pytest.fail("host authorization must not be reached")) + with pytest.raises(ValueError, match="exact already imported"): + fixture.run_fixture(**inputs) + + +def test_linked_credential_stops_before_host_authorization(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + linked = tmp_path / "linked-token" + linked.symlink_to(inputs["source_token_file"]) + inputs["source_token_file"] = linked + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: pytest.fail("host authorization must not be reached")) + with pytest.raises(ValueError, match="canonical absolute"): + fixture.run_fixture(**inputs) + + +def test_no_child_or_admission_oracle_rejects_any_native_store(tmp_path: Path) -> None: + state, journal = tmp_path / "state", tmp_path / "journal" + state.mkdir() + journal.mkdir() + (state / ".supervisor-owner.lock").write_bytes(b"") + (state / "logs").mkdir() + generation = Generation("target", tmp_path, "a" * 64, []) + fixture._no_target_child(generation, running=False) + (state / "management-child.sqlite3").write_bytes(b"not allowed") + with pytest.raises(AssertionError, match="native consumer state"): + fixture._no_target_child(generation, running=False) + (state / "management-child.sqlite3").unlink() + (journal / "management-operations.sqlite3").write_bytes(b"not allowed") + with pytest.raises(AssertionError, match="admitted an operation"): + fixture._no_target_child(generation, running=False) + + +def test_original_fingerprint_binds_inodes_and_bytes(tmp_path: Path) -> None: + path = tmp_path / "native-original" + path.write_bytes(b"original") + before = fixture._snapshot((tmp_path,)) + assert fixture._snapshot((tmp_path,)) == before + other = tmp_path / "replacement" + other.write_bytes(path.read_bytes()) + os.replace(other, path) + assert fixture._snapshot((tmp_path,)) != before + + +def _workers(root: Path) -> None: + for channel in ("beta", "release", "agent-table"): + parent = root / channel / "supervisor" + parent.mkdir(parents=True) + (parent / "state.json").write_bytes( + canonical({"schema_version": 1, "operations": {}, "selected_runtime_id": "bootstrap", "worker": {"pid": 2, "start_token": "1000", "process_run_id": channel + "-run", "deployment_generation": channel + "-deployment", "runtime_id": "bootstrap", "binding_observed_at": 1.0}}) + ) + for name in ("compose.yml", "beta-secret", "release-secret", "agent-table-secret"): + (root / name).write_bytes(b"original-component-input") + + +def test_worker_claim_oracle_preserves_effects_and_excludes_only_observation_refresh(tmp_path: Path) -> None: + _workers(tmp_path) + baseline = fixture._worker_claims(tmp_path) + path = tmp_path / "beta" / "supervisor" / "state.json" + state = json.loads(path.read_bytes()) + state["worker"]["binding_observed_at"] = 2.0 + replacement = path.with_name("replacement") + replacement.write_bytes(canonical(state)) + os.replace(replacement, path) + assert fixture._worker_claims(tmp_path) == baseline + state["operations"]["new-operation"] = {"status": "running"} + path.write_bytes(canonical(state)) + assert fixture._worker_claims(tmp_path) != baseline + + +def test_worker_claim_oracle_requires_real_nonempty_ownership(tmp_path: Path) -> None: + _workers(tmp_path) + path = tmp_path / "beta" / "supervisor" / "state.json" + state = json.loads(path.read_bytes()) + state["worker"]["process_run_id"] = None + path.write_bytes(canonical(state)) + with pytest.raises(AssertionError, match="ownership evidence"): + fixture._worker_claims(tmp_path) + + +def _durable(root: Path) -> tuple[str, str]: + # Deliberately synthetic pure oracle input; the VM driver obtains these + # records only from the actual production issuer and genuine peer logs. + inventory = canonical({"component": "inventory"}) + (root / "inventory.json").write_bytes(inventory) + inventory_sha = hashlib.sha256(inventory).hexdigest() + evidence = canonical({"lineage": {"unavailable": [], "genesis": {"component": "origin"}}, "inspection": {"unresolved_targets": 1, "historical_keys": 3, "launch_authority": False}, "target": {"receipt": {"source_sha": "a" * 40}}}).decode() + binding = {"evidence_json": evidence, "evidence_sha256": hashlib.sha256(evidence.encode()).hexdigest()} + previous = "" + for sequence, phase in enumerate(("reserved", "intent", "consumed", "receipt")): + record = canonical({"schema_version": 1, "phase": phase, "sequence": sequence, "previous": previous, "inventory_sha256": inventory_sha, "binding": binding}) + (root / (phase + ".json")).write_bytes(record) + head = canonical({"schema_version": 1, "phase": phase, "record_sha256": hashlib.sha256(record).hexdigest(), "previous": previous, "inventory_sha256": inventory_sha}) + (root / (phase + "-head.json")).write_bytes(head) + previous = hashlib.sha256(head).hexdigest() + return "One consumed recovery epoch grant send completed", "Recovery custody acknowledged" + + +def test_durable_oracle_requires_all_heads_and_actual_consumer_acknowledgment(tmp_path: Path) -> None: + issuer, consumer = _durable(tmp_path) + result = fixture.inspect_durable_result(tmp_path, issuer_log=issuer, consumer_log=consumer) + assert result["unresolved_targets"] == 1 and result["historical_keys"] == 3 + with pytest.raises(AssertionError): + fixture.inspect_durable_result(tmp_path, issuer_log=issuer, consumer_log="consumer waiting") + (tmp_path / "receipt-head.json").unlink() + with pytest.raises(FileNotFoundError): + fixture.inspect_durable_result(tmp_path, issuer_log=issuer, consumer_log=consumer) + + +def test_durable_oracle_rejects_a_changed_original_record(tmp_path: Path) -> None: + issuer, consumer = _durable(tmp_path) + path = tmp_path / "consumed.json" + value = json.loads(path.read_bytes()) + value["binding"]["evidence_sha256"] = "b" * 64 + path.write_bytes(canonical(value)) + with pytest.raises(AssertionError): + fixture.inspect_durable_result(tmp_path, issuer_log=issuer, consumer_log=consumer) + + +def _result() -> dict[str, object]: + return { + "schema": fixture.RESULT_SCHEMA, + "status": "passed", + "cases": sorted(fixture.REQUIRED_CASES), + "no_child_started": True, + "preserved_originals": True, + "preserved_workers": True, + "original_outcomes": {"C": "launched", "D": "never-created"}, + "durable": {"record_sha256": {name: "a" * 64 for name in ("reserved", "intent", "consumed", "receipt")}}, + } + + +@pytest.mark.parametrize("case", sorted(fixture.REQUIRED_CASES)) +def test_native_result_cannot_omit_any_required_case(case: str) -> None: + result = _result() + fixture.validate_result(result) + result["cases"].remove(case) + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("field", ["no_child_started", "preserved_originals", "preserved_workers"]) +def test_native_result_cannot_claim_success_without_preservation(field: str) -> None: + result = _result() + result[field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("scenario,phase", [("kill-after-intent", "intent"), ("kill-after-consume", "consumed"), ("lost-reply", "receipt")]) +def test_interruption_result_requires_the_actual_boundary_and_no_grant(scenario: str, phase: str) -> None: + result = _result() + result.update(scenario=scenario, cases=sorted(fixture.required_cases(scenario)), durable={"phase": phase, "consumer_grant_acknowledged": False}) + fixture.validate_result(result) + result["durable"]["consumer_grant_acknowledged"] = True + with pytest.raises(AssertionError): + fixture.validate_result(result) + result["durable"] = {"phase": "reserved", "consumer_grant_acknowledged": False} + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +def test_unknown_native_scenario_is_not_a_success_alias() -> None: + with pytest.raises(ValueError, match="unsupported"): + fixture.required_cases("unreviewed-boundary") diff --git a/tests/test_management_recovery_epoch_join.py b/tests/test_management_recovery_epoch_join.py new file mode 100644 index 000000000..fd2b9cf02 --- /dev/null +++ b/tests/test_management_recovery_epoch_join.py @@ -0,0 +1,426 @@ +"""Authority-join component contracts, not original or live host authority. + +The lineage and containment dataclasses below are explicit component values. +Their agreement tests the issuer's cross-producer joins, not their provenance. +Consumer-scope tests additionally use real protected files and held kernel FDs; +no fixture result substitutes for production Docker or current T observations. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import os +import pwd +import stat +import tempfile +from contextlib import ExitStack, contextmanager +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Iterator + +import pytest + +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.containment_observation import DockerObservation +from auto_coder.management.containment_route_manifest import ApprovedRouteManifest, DeploymentRootBinding, RouteManifestObservation +from auto_coder.management.original_attempt_evidence import OriginalGeneration, OriginalLineageEvidence +from auto_coder.management.original_attempts import HistoryBinding, canonical, source_identity +from auto_coder.management.original_genesis import GenesisAnchor, GenesisRoot +from auto_coder.management.recovery_containment import RecoveryContainmentFacts, RecoveryContainmentReport, RecoveryTargetFact +from auto_coder.management.recovery_containment_model import NamedDescriptor, RecoveryEnrollment, RecoveryEnrollmentObservation, RecoveryFile, RecoveryLock, RecoveryOriginal, RecoveryResources, RecoveryTarget +from auto_coder.management.recovery_containment_profile import RecoveryManagerFact +from auto_coder.management.recovery_containment_resources import RecoveryResourceObservation +from auto_coder.management.recovery_epoch import _check_consumer_scope, join_lineage +from auto_coder.management.recovery_epoch_authorization import PEER_ROOT, EpochAuthorization +from auto_coder.management.recovery_epoch_consumer import ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource +from auto_coder.management.recovery_epoch_peer import HeldFence +from auto_coder.management.recovery_epoch_resources import EpochResources +from auto_coder.management.recovery_epoch_state import EpochInventory, EpochUnavailable, issuer_lifetime + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") +_BOOT = "00000000-0000-4000-8000-000000000001" +_IMAGE = "sha256:" + "a" * 64 +_OAUTH = "/component/oauth" + + +def component_file(path: str, inode: int) -> RecoveryFile: + return RecoveryFile(path, format(inode, "064x"), 1, inode) + + +def component_generation(index: int, source: str, *, never: bool = False) -> tuple[OriginalGeneration, RecoveryOriginal, RecoveryManagerFact]: + manager, proxy = format(2 * index + 1, "064x"), format(2 * index + 2, "064x") + base = "/component/generation-" + str(index) + state, journal = base + "/state", base + "/journal" + bootstrap, management = component_file(base + "/config.json", 100 + index), component_file(base + "/management.yaml", 200 + index) + owner = None if never else RecoveryLock(state + "/.supervisor-owner.lock", 1, 300 + index) + mounts = ( + DeploymentRootBinding(manager, "/management-child-state", state, True, 1, 400 + index, stat.S_IFDIR | 0o700), + DeploymentRootBinding(manager, "/management-journal", journal, True, 1, 500 + index, stat.S_IFDIR | 0o700), + DeploymentRootBinding(manager, "/management-oauth", _OAUTH, True, 1, 600, stat.S_IFDIR | 0o700), + ) + generation = OriginalGeneration( + f"00000000-0000-4000-8000-{index + 10:012d}", + f"instance-{index}", + manager, + _IMAGE, + bootstrap.sha256, + management.sha256, + state, + 1, + 400 + index, + journal, + 1, + 500 + index, + _OAUTH, + 1, + 600, + owner.path if owner else "", + owner.device if owner else 0, + owner.inode if owner else 0, + "never-created" if never else "unknown", + (), + proxy if index == 0 else "", + _IMAGE if index == 0 else "", + ) + original = RecoveryOriginal( + f"generation-{index}", + generation.instance_id, + manager, + _IMAGE, + proxy, + _IMAGE, + "never-started" if never else "started", + "" if never else _BOOT, + bootstrap, + management, + component_file(base + "/Caddyfile", 700 + index), + mounts, + owner, + None if never else component_file(base + "/prior.json", 800 + index), + None if never else OriginalNetworkPin(42, "12345", _BOOT, 10 + index, f"net:[{900 + index}]", 4, 900 + index), + ) + fact = RecoveryManagerFact(original.generation_id, generation.instance_id, manager, _IMAGE, original.state, source, "b" * 64, bootstrap.sha256, management.sha256, state, journal, _OAUTH, owner, 18080 + index, mounts) + return generation, original, fact + + +def component_docker(identifier: str) -> DockerObservation: + return DockerObservation("engine-one", identifier, _IMAGE, "c" * 64, "no", 0, False, "exited", 0, "old-start", "old-stop", 0, "d" * 64) + + +@dataclass(frozen=True) +class JoinCase: + authorization: EpochAuthorization + inventory: EpochInventory + enrollment: RecoveryEnrollmentObservation + lineage: OriginalLineageEvidence + containment: RecoveryContainmentReport + + def check(self) -> None: + join_lineage(self.authorization, self.inventory, self.enrollment, self.lineage, self.containment) + + def scope(self, selected: RecoveryEnrollment) -> JoinCase: + enrollment = replace(self.enrollment, enrollment=selected) + return replace(self, enrollment=enrollment, containment=replace(self.containment, facts=replace(self.containment.facts, enrollment=enrollment))) + + def managers(self, values: tuple[RecoveryManagerFact, ...]) -> JoinCase: + return replace(self, containment=replace(self.containment, facts=replace(self.containment.facts, managers=values))) + + +def component_join() -> JoinCase: + source = source_identity() + history = HistoryBinding("/component/history", "deployment-one", "history-one", source) + first, second, absent = (component_generation(index, source, never=index == 2) for index in range(3)) + target_id = "f" * 64 + target_roots = ( + DeploymentRootBinding(target_id, "/management-child-state", "/component/target/state", True, 1, 1001, stat.S_IFDIR | 0o700), + DeploymentRootBinding(target_id, "/management-journal", "/component/target/journal", True, 1, 1002, stat.S_IFDIR | 0o700), + DeploymentRootBinding(target_id, "/management-oauth", _OAUTH, True, 1, 600, stat.S_IFDIR | 0o700), + ) + target = RecoveryTarget("target-one", target_id, _IMAGE, component_file("/component/target/config.json", 1101), component_file("/component/target/management.yaml", 1102), target_roots, RecoveryLock("/component/target/state/.supervisor-owner.lock", 1, 1103), 18090) + selected = RecoveryEnrollment("approved-management-recovery-containment-v1", history.deployment, "enrollment-one", "authorization-one", "ordinary-recovery-v1", "engine-one", source, (first[1], second[1]), target, (), True) + enrollment = RecoveryEnrollmentObservation(selected, "1" * 64, 1, 1200, 100) + genesis = GenesisAnchor("ordinary-original-genesis-v1", history, "old-authorization", first[0].instance_id, "engine-one", first[0].management_id, _IMAGE, "2" * 64, first[1].proxy_id, _IMAGE, "3" * 64, (), (GenesisRoot("oauth", _OAUTH, "/management-oauth", 1, 600),), "{}") + lineage = OriginalLineageEvidence(history, genesis=genesis, frontier="4" * 64, generations=(first[0], second[0], absent[0])) + target_fact = RecoveryManagerFact( + "target", target.instance_id, target_id, _IMAGE, "prelaunch-target", source, "5" * 64, target.bootstrap_config.sha256, target.management_config.sha256, "/component/target/state", "/component/target/journal", _OAUTH, target.owner, target.published_port, target_roots + ) + docker = tuple(component_docker(identifier) for identifier in selected.container_ids) + manifest = ApprovedRouteManifest("approved-management-recovery-routes-v1", selected.deployment_id, selected.enrollment_id, selected.authorization_id, selected.profile, enrollment.sha256, True, True, (), selected.container_ids, (), (), (), ()) + routes = RouteManifestObservation(manifest, "6" * 64, 1, 1300, 100, (), (), (), selected.container_ids) + facts = RecoveryContainmentFacts( + enrollment, + source, + docker, + (first[2], second[2], target_fact), + (), + (), + RecoveryTargetFact(target_id, 101, "12345", _BOOT, "net:[1400]", "pid:[1401]", "/component/target-cgroup", 1, 1402, 1, target.owner.inode, target.published_port), + RecoveryResourceObservation(102, "12346", _BOOT, ()), + routes, + (), + (), + ) + containment = RecoveryContainmentReport("current-recovery-containment-v1", "observed-current-recovery-containment", "00000000-0000-4000-8000-000000000050", 102, "12346", _BOOT, 100, 200, facts, "7" * 64) + authorization = EpochAuthorization( + "management-recovery-authorization-v1", + selected.deployment_id, + selected.authorization_id, + "new-request", + "new-preparation", + target_id, + selected.enrollment_id, + history, + component_file("/component/target/consumer.json", 1500), + component_file("/component/target/READY.json", 1501), + "/component/source", + "/component/source-token", + "/usr/bin/python3", + "/usr/bin/uv", + "0.13.6", + "a" * 40, + 1, + 1600, + ) + inventory = EpochInventory( + 2, "/var/lib/auto-coder/recovery-epoch", 1, 1700, 1, 1701, selected.deployment_id, selected.authorization_id, (selected.enrollment_id,), authorization.request_id, authorization.preparation_id, target_id, source.removeprefix("sha256:"), target.bootstrap_config.sha256, "8" * 64 + ) + return JoinCase(authorization, inventory, enrollment, lineage, containment) + + +def test_agreeing_joins_preserve_unknowns_and_proven_absent_never_created_branch() -> None: + case = component_join() + before = asdict(case.lineage) + case.check() + assert asdict(case.lineage) == before + assert [item.outcome for item in case.lineage.generations] == ["unknown", "unknown", "never-created"] + assert case.lineage.generations[-1].management_id not in case.containment.docker_ids + + +@pytest.mark.parametrize("change", ["missing-genesis", "missing-generations", "unavailable", "foreign-history", "reused-request", "reused-authorization"]) +def test_original_authority_must_be_complete_and_new_recovery_identity_distinct(change: str) -> None: + case = component_join() + if change == "missing-genesis": + case = replace(case, lineage=replace(case.lineage, genesis=None)) + elif change == "missing-generations": + case = replace(case, lineage=replace(case.lineage, generations=())) + elif change == "unavailable": + case = replace(case, lineage=replace(case.lineage, unavailable=("original stores absent",))) + elif change == "foreign-history": + case = replace(case, lineage=replace(case.lineage, binding=replace(case.lineage.binding, history="foreign"))) + elif change == "reused-request": + case = replace(case, authorization=replace(case.authorization, request_id=case.lineage.generations[0].request_id)) + else: + assert case.lineage.genesis is not None + case = replace(case, authorization=replace(case.authorization, authorization_id=case.lineage.genesis.authorization_id)) + with pytest.raises(EpochUnavailable): + case.check() + + +@pytest.mark.parametrize("change", ["unavailable-status", "claims-grant", "claims-history", "claims-admission", "stale-enrollment"]) +def test_containment_is_the_exact_current_nonauthorizing_scope(change: str) -> None: + case = component_join() + if change == "stale-enrollment": + stale = replace(case.enrollment, inode=case.enrollment.inode + 1) + case = replace(case, containment=replace(case.containment, facts=replace(case.containment.facts, enrollment=stale))) + else: + field, value = {"unavailable-status": ("status", "unavailable"), "claims-grant": ("grant_issued", True), "claims-history": ("history_asserted", True), "claims-admission": ("admission_created", True)}[change] + case = replace(case, containment=replace(case.containment, **{field: value})) + with pytest.raises(EpochUnavailable, match="non-authorizing"): + case.check() + + +@pytest.mark.parametrize("change", ["deployment", "source", "inventory-source", "target", "target-config", "engine"]) +def test_current_source_target_and_engine_must_join_original_and_fixed_inventory(change: str) -> None: + case = component_join() + if change in ("deployment", "source", "engine"): + field, value = {"deployment": ("deployment_id", "foreign"), "source": ("source_identity", "sha256:" + "0" * 64), "engine": ("engine_id", "another-engine")}[change] + case = case.scope(replace(case.enrollment.enrollment, **{field: value})) + else: + field = {"inventory-source": "source_sha256", "target": "target_id", "target-config": "config_sha256"}[change] + case = replace(case, inventory=replace(case.inventory, **{field: "0" * 64})) + with pytest.raises(EpochUnavailable): + case.check() + + +@pytest.mark.parametrize("change", ["omitted-lineage", "omitted-observation", "omitted-started-scope", "duplicate-lineage", "duplicate-observation"]) +def test_no_original_generation_or_current_manager_can_disappear_or_alias(change: str) -> None: + case = component_join() + if change == "omitted-lineage": + case = replace(case, lineage=replace(case.lineage, generations=(case.lineage.generations[0], case.lineage.generations[2]))) + elif change == "duplicate-lineage": + case = replace(case, lineage=replace(case.lineage, generations=(*case.lineage.generations, case.lineage.generations[0]))) + elif change == "duplicate-observation": + case = case.managers((*case.containment.managers, case.containment.managers[0])) + else: + case = case.managers((case.containment.managers[0], case.containment.managers[-1])) + if change == "omitted-started-scope": + case = case.scope(replace(case.enrollment.enrollment, originals=case.enrollment.enrollment.originals[:1])) + with pytest.raises(EpochUnavailable): + case.check() + + +@pytest.mark.parametrize("change", ["bootstrap", "proxy", "owner", "root-path", "root-inode", "missing-root"]) +def test_original_native_configuration_owner_and_actual_root_objects_must_match(change: str) -> None: + case = component_join() + original, fact = case.enrollment.enrollment.originals[0], case.containment.managers[0] + if change == "bootstrap": + original = replace(original, bootstrap_config=replace(original.bootstrap_config, sha256="0" * 64)) + elif change == "proxy": + original = replace(original, proxy_id="0" * 64) + elif change == "owner": + assert original.owner is not None + original = replace(original, owner=replace(original.owner, inode=original.owner.inode + 1)) + elif change == "root-path": + fact = replace(fact, state_root="/component/different-root") + elif change == "root-inode": + fact = replace(fact, mounts=(replace(fact.mounts[0], inode=fact.mounts[0].inode + 1), *fact.mounts[1:])) + else: + fact = replace(fact, mounts=fact.mounts[1:]) + case = case.scope(replace(case.enrollment.enrollment, originals=(original, *case.enrollment.enrollment.originals[1:]))).managers((fact, *case.containment.managers[1:])) + with pytest.raises(EpochUnavailable): + case.check() + + +@pytest.mark.parametrize("change", ["docker-object-exists", "owner-exists", "outcome-unknown"]) +def test_absent_branch_requires_original_never_created_boundary_without_current_object(change: str) -> None: + case = component_join() + absent = case.lineage.generations[-1] + if change == "docker-object-exists": + case = replace(case, containment=replace(case.containment, facts=replace(case.containment.facts, docker=(*case.containment.facts.docker, component_docker(absent.management_id))))) + else: + absent = replace(absent, **({"owner_inode": 9999} if change == "owner-exists" else {"outcome": "unknown"})) + case = replace(case, lineage=replace(case.lineage, generations=(*case.lineage.generations[:-1], absent))) + with pytest.raises(EpochUnavailable, match="absent"): + case.check() + + +@pytest.mark.parametrize("change", ["oauth-path", "oauth-inode", "state-reused", "journal-nested", "state-parent"]) +def test_target_preserves_original_oauth_and_keeps_new_roots_disjoint(change: str) -> None: + case = component_join() + target = case.containment.managers[-1] + if change == "oauth-path": + target = replace(target, oauth_root="/component/other-oauth") + elif change == "oauth-inode": + target = replace(target, mounts=(*target.mounts[:-1], replace(target.mounts[-1], inode=601))) + elif change == "state-reused": + target = replace(target, state_root=case.lineage.generations[0].state_root) + elif change == "journal-nested": + target = replace(target, journal_root=case.lineage.generations[-1].journal_root + "/nested") + else: + target = replace(target, state_root="/component") + with pytest.raises(EpochUnavailable): + case.managers((*case.containment.managers[:-1], target)).check() + + +@dataclass +class ScopeCase: + authorization: EpochAuthorization + enrollment: RecoveryEnrollment + resources: EpochResources + configuration: ConsumerConfiguration + path: Path + + def write_configuration(self, configuration: ConsumerConfiguration) -> None: + self.configuration = configuration + self.path.write_bytes(canonical(asdict(configuration))) + info = self.path.stat() + self.authorization = replace(self.authorization, consumer_config=RecoveryFile(str(self.path), hashlib.sha256(self.path.read_bytes()).hexdigest(), info.st_dev, info.st_ino)) + + def check(self) -> None: + _check_consumer_scope(self.authorization, self.enrollment, self.resources) + + +@contextmanager +def component_scope() -> Iterator[ScopeCase]: + with tempfile.TemporaryDirectory(prefix="ac-recovery-join-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory, ExitStack() as held: + root = Path(directory) + history_root = root / "history" + history_root.mkdir(mode=0o700) + paths = {"history": history_root / "history.lock", "owner-0": root / "owner.lock", "controller-0": root / "controller.lock", "epoch": root / "epoch.lock", "target-owner": root / "target.lock"} + descriptors = {} + for name, path in paths.items(): + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + held.callback(os.close, descriptor) + descriptors[name] = descriptor + if name != "target-owner": + fcntl.flock(descriptor, (fcntl.LOCK_SH if name in ("history", "owner-0") else fcntl.LOCK_EX) | fcntl.LOCK_NB) + identities = {name: os.fstat(descriptor) for name, descriptor in descriptors.items()} + fixture = component_join() + original = replace(fixture.enrollment.enrollment.originals[0], owner=RecoveryLock(str(paths["owner-0"]), identities["owner-0"].st_dev, identities["owner-0"].st_ino)) + target = fixture.enrollment.enrollment.target + mounts = [DeploymentRootBinding(target.management_id, "/management-recovery", str(PEER_ROOT), False, 1, 1600, stat.S_IFDIR | 0o700)] + original_paths = {"history": "/management-recovery-history/history.lock", "owner-0": "/management-recovery-originals/0/owner.lock"} + mounts.extend(DeploymentRootBinding(target.management_id, destination, str(paths[name]), False, identities[name].st_dev, identities[name].st_ino, stat.S_IFREG | 0o600) for name, destination in original_paths.items()) + target = replace(target, mounts=tuple(mounts), owner=RecoveryLock(str(paths["target-owner"]), identities["target-owner"].st_dev, identities["target-owner"].st_ino)) + enrollment = replace(fixture.enrollment.enrollment, originals=(original,), target=target) + fences = tuple(HeldFence(name, descriptors[name], "original-sh" if name in original_paths else "target-ex" if name == "target-owner" else "transferred-ex") for name in paths) + boot, pid, start = issuer_lifetime() + resources = EpochResources(descriptors["history"], RecoveryResources(pid, start, boot, (NamedDescriptor(original.generation_id, descriptors["owner-0"]),), (), ()), fences, ()) + configuration = ConsumerConfiguration( + 1, + fixture.authorization.request_id, + fixture.authorization.authorization_id, + target.management_id, + 0, + 1, + 1600, + tuple(OriginalFence(name, original_paths[name], identities[name].st_dev, identities[name].st_ino) for name in original_paths), + TargetOwner("target-owner", identities["target-owner"].st_dev, identities["target-owner"].st_ino), + tuple(TransferredResource(name, "transferred-ex", identities[name].st_dev, identities[name].st_ino) for name in ("controller-0", "epoch")), + ) + path = root / "consumer.json" + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + case = ScopeCase(replace(fixture.authorization, history=replace(fixture.authorization.history, root=str(history_root))), enrollment, resources, configuration, path) + case.write_configuration(configuration) + yield case + + +def test_consumer_scope_joins_exact_real_fds_and_pinned_read_only_mounts() -> None: + with component_scope() as case: + case.check() + + +@pytest.mark.parametrize("change", ["missing-peer", "writable-peer", "foreign-peer-source", "replaced-peer-inode", "duplicate-peer", "writable-original", "foreign-original-source", "replaced-original-inode"]) +def test_consumer_scope_requires_exact_read_only_endpoint_and_original_mounts(change: str) -> None: + with component_scope() as case: + mounts = list(case.enrollment.target.mounts) + if change == "missing-peer": + mounts.pop(0) + elif change == "duplicate-peer": + mounts.append(mounts[0]) + else: + index = 0 if "peer" in change else 1 + if change.startswith("writable"): + mounts[index] = replace(mounts[index], writable=True) + elif change.startswith("foreign"): + mounts[index] = replace(mounts[index], source="/component/foreign") + else: + mounts[index] = replace(mounts[index], inode=mounts[index].inode + 1) + case.enrollment = replace(case.enrollment, target=replace(case.enrollment.target, mounts=tuple(mounts))) + with pytest.raises(EpochUnavailable, match="read-only"): + case.check() + + +@pytest.mark.parametrize("change", ["foreign-target", "omitted-original", "wrong-original-path", "target-owner-inode", "omitted-controller", "transfer-inode", "transfer-kind"]) +def test_consumer_scope_cannot_omit_or_retarget_acknowledged_authority(change: str) -> None: + with component_scope() as case: + configuration = case.configuration + if change == "foreign-target": + configuration = replace(configuration, target_id="e" * 64) + elif change == "omitted-original": + configuration = replace(configuration, original_fences=configuration.original_fences[:1]) + elif change == "wrong-original-path": + configuration = replace(configuration, original_fences=(replace(configuration.original_fences[0], path="/component/foreign"), *configuration.original_fences[1:])) + elif change == "target-owner-inode": + configuration = replace(configuration, target_owner=replace(configuration.target_owner, inode=configuration.target_owner.inode + 999)) + elif change == "omitted-controller": + configuration = replace(configuration, transferred_resources=configuration.transferred_resources[1:]) + elif change == "transfer-inode": + configuration = replace(configuration, transferred_resources=(replace(configuration.transferred_resources[0], inode=configuration.transferred_resources[0].inode + 999), *configuration.transferred_resources[1:])) + else: + configuration = replace(configuration, transferred_resources=(replace(configuration.transferred_resources[0], kind="network-namespace"), *configuration.transferred_resources[1:])) + case.write_configuration(configuration) + with pytest.raises(EpochUnavailable): + case.check() diff --git a/tests/test_management_recovery_epoch_observability.py b/tests/test_management_recovery_epoch_observability.py new file mode 100644 index 000000000..c6b0ca872 --- /dev/null +++ b/tests/test_management_recovery_epoch_observability.py @@ -0,0 +1,71 @@ +"""Real durable-phase emitters; no grant, operation, or readiness evidence.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from auto_coder.management import recovery_epoch_state as epoch + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + +_EMIT = r""" +import json,os,sys +from dataclasses import asdict +from pathlib import Path +from auto_coder.logger_config import setup_logger +from auto_coder.management import recovery_epoch_state as epoch +from auto_coder.management.original_attempts import canonical +root=Path(sys.argv[1])/'epoch' +root.mkdir(mode=0o700) +fd=os.open(root/'epoch.lock',os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) +os.close(fd) +directory,lock=root.stat(),(root/'epoch.lock').stat() +inventory=epoch.EpochInventory(1,str(root),directory.st_dev,directory.st_ino,lock.st_dev,lock.st_ino,'component-deployment','component-authorization',('component-enrollment',),'component-request','component-preparation','a'*64,'b'*64,'c'*64) +fd=os.open(root/'inventory.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) +with os.fdopen(fd,'wb') as stream: + stream.write(canonical(asdict(inventory))) +boot,pid,start=epoch.issuer_lifetime() +binding=epoch.EpochBinding(inventory.deployment_id,inventory.authorization_id,inventory.enrollment_ids,inventory.request_id,inventory.preparation_id,inventory.target_id,inventory.source_sha256,inventory.config_sha256,'d'*64,boot,pid,start) +setup_logger(stream=sys.stdout,log_file=str(root.parent/'events.log')) +sink=epoch.logger.add(root.parent/'events.jsonl',serialize=True,rotation='1 MB',enqueue=False,diagnose=False,filter=lambda record:'recovery_epoch_phase' in record['extra']) +try: + with epoch._hold_recovery_epoch(root,expected_uid=os.geteuid()) as state: + state.reserve(binding) + state.record_intent(binding) + receipt=state.consume(binding) + epoch.logger.complete() +finally: + epoch.logger.remove(sink) +print(json.dumps(asdict(receipt),sort_keys=True),flush=True) +""" + + +def test_actual_durable_phase_logs_reach_stdout_and_file_without_success_or_readiness(tmp_path: Path) -> None: + result = subprocess.run([sys.executable, "-B", "-c", _EMIT, str(tmp_path)], text=True, capture_output=True, check=True, timeout=20) + emitted = [json.loads(line)["record"] for line in (tmp_path / "events.jsonl").read_text().splitlines()] + phases = ("reserved", "intent", "consumed", "receipt") + assert [record["extra"]["recovery_epoch_phase"] for record in emitted] == list(phases) + persisted_text = (tmp_path / "events.log").read_text() + for phase, record in zip(phases, emitted): + durable = json.loads((tmp_path / "epoch" / (phase + ".json")).read_text()) + assert durable["phase"] == phase and durable["binding"]["request_id"] == "component-request" + assert record["file"]["path"] == str(Path(epoch.__file__).resolve()) + assert record["file"]["name"] == "recovery_epoch_state.py" and record["function"] == "_append" and record["line"] > 0 + assert record["level"]["name"] == "INFO" + extra = record["extra"] + assert extra["request_id"] == "component-request" and extra["target_id"] == "a" * 64 + assert extra["grant_delivered"] is False + assert not {"operation_id", "event_type", "status", "outcome", "application_ready", "current_ready", "ready"} & set(extra) + expected_message = f"Recovery epoch {phase} durably recorded; no grant delivery observed" + assert record["message"] == expected_message + assert expected_message in result.stdout and expected_message in persisted_text + receipt = json.loads(result.stdout.splitlines()[-1]) + assert receipt["grant_delivered"] is False + assert set(receipt) == {"inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256", "grant_delivered"} + assert os.stat(tmp_path / "epoch" / "receipt.json").st_size > 0 diff --git a/tests/test_management_recovery_epoch_peer.py b/tests/test_management_recovery_epoch_peer.py new file mode 100644 index 000000000..2684fe959 --- /dev/null +++ b/tests/test_management_recovery_epoch_peer.py @@ -0,0 +1,368 @@ +"""Real local processes and kernel custody, not Docker/recovery grant evidence.""" + +from __future__ import annotations + +import array +import fcntl +import json +import os +import select +import socket +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Iterator + +import pytest + +from auto_coder.management.recovery_epoch_peer import ExpectedPeer, HeldFence, HeldResource, PeerCustody, PeerSession, PeerUnavailable, inspect_held_resource + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + +# The consumer speaks the public wire schema with real independent opens and +# flocks. No product observer or kernel authority is replaced by this fixture. +_CONSUMER = r""" +import array, fcntl, json, os, secrets, select, socket, sys, time +mailbox_fd, paths_json, mode, inherited_json = sys.argv[1:] +paths, inherited = json.loads(paths_json), json.loads(inherited_json) +owned = {} +for name, operation in (('original', fcntl.LOCK_SH), ('target', fcntl.LOCK_EX)): + if (name == 'original' and mode == 'inherited-original') or (name == 'target' and mode == 'inherited-target'): + owned[name] = inherited[name] + else: + owned[name] = os.open(paths[name], os.O_RDONLY | os.O_NOFOLLOW) + if not (name == 'original' and mode == 'unlocked-original'): + fcntl.flock(owned[name], operation | fcntl.LOCK_NB) +mailbox = socket.socket(fileno=int(mailbox_fd)) +connection, issuer_endpoint = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) +mailbox.sendmsg([b'p'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [issuer_endpoint.fileno()]))]) +issuer_endpoint.close() +mailbox.close() +connection.settimeout(10) +request = { + 'schema': 'management-recovery-peer-challenge-v1', + 'request_id': 'request-one', 'authorization_id': 'authorization-one', + 'target_id': 'target-one', 'challenge': secrets.token_hex(32), +} +if mode == 'wrong-target': request['target_id'] = 'target-two' +if mode == 'bad-challenge': request['challenge'] = 'old' +wire = json.dumps(request).encode() + b'\n' +if mode == 'oversized-challenge': wire = b' ' * 65537 + wire +if mode == 'duplicate-challenge': wire = wire.rstrip()[:-1] + b',"challenge":"' + b'b' * 64 + b'"}\n' +if mode == 'rights-challenge': + connection.sendmsg([wire], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [owned['original']]))]) +else: + connection.sendall(wire) +print('challenge-sent', flush=True) +if mode in ('wrong-target', 'bad-challenge', 'oversized-challenge', 'duplicate-challenge', 'rights-challenge'): + sys.stdin.readline() + raise SystemExit(0) +if mode == 'queued-only': + if not select.select([connection], [], [], 10)[0]: raise RuntimeError('offer was never queued') + print('offer-queued', flush=True) + sys.stdin.readline() + raise SystemExit(0) +received = [] +content = bytearray() +while b'\n' not in content: + data, ancillary, flags, _ = connection.recvmsg(65537, socket.CMSG_SPACE(256), socket.MSG_CMSG_CLOEXEC) + if not data or flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC): raise RuntimeError('incomplete offer') + content.extend(data) + for level, kind, payload in ancillary: + if (level, kind) != (socket.SOL_SOCKET, socket.SCM_RIGHTS): raise RuntimeError('unexpected control') + values = array.array('i'); values.frombytes(payload); received.extend(values) +offer = json.loads(content) +descriptors = [] +for resource in offer['resources']: + index = resource['transfer_index'] + descriptor = received[index] if index is not None else owned[resource['name']] + owned[resource['name']] = descriptor + descriptors.append({'name': resource['name'], 'fd': descriptor}) +print('offer-received', flush=True) +if sys.stdin.readline().strip() != 'receipt': raise SystemExit(0) +receipt = {key: offer[key] for key in ('request_id', 'authorization_id', 'target_id', 'challenge', 'receipt_nonce')} +receipt.update(schema='management-recovery-custody-receipt-v1', descriptors=descriptors) +if mode == 'omitted': descriptors.pop() +if mode == 'duplicate-fd': descriptors[-1]['fd'] = descriptors[0]['fd'] +if mode == 'duplicate-name': descriptors[-1]['name'] = descriptors[0]['name'] +if mode == 'unknown-name': descriptors[-1]['name'] = 'undeclared' +if mode == 'stale-nonce': receipt['receipt_nonce'] = '0' * 64 +if mode == 'wrong-fd': descriptors[2]['fd'] = os.open(paths['other'], os.O_RDONLY) +if mode == 'reopened-controller': descriptors[2]['fd'] = os.open(paths['controller'], os.O_RDONLY) +if mode == 'closed-controller': os.close(owned['controller']) +if mode == 'boolean-fd': descriptors[0]['fd'] = True +wire = json.dumps(receipt).encode() + b'\n' +if mode == 'rights-receipt': + connection.sendmsg([wire], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [owned['original']]))]) +elif mode == 'drip-receipt': + for byte in wire: + try: connection.sendall(bytes([byte])) + except BrokenPipeError: break + time.sleep(0.04) +else: + connection.sendall(wire) +print('receipt-sent', flush=True) +for command in sys.stdin: + command = command.strip() + if command == 'exit': break + if command.startswith('close-'): + os.close(owned[command.removeprefix('close-')]) + elif command.startswith('unlock-'): + fcntl.flock(owned[command.removeprefix('unlock-')], fcntl.LOCK_UN) + else: raise RuntimeError('unsupported fixture command') + print(command, flush=True) +""" + + +@dataclass +class PeerCase: + process: subprocess.Popen[str] + connection: socket.socket + expected: ExpectedPeer + fences: tuple[HeldFence, ...] + resources: tuple[HeldResource, ...] + paths: dict[str, str] + + def line(self, expected: str) -> None: + assert self.process.stdout is not None + assert select.select([self.process.stdout], [], [], 10)[0], "peer did not reach the observable protocol barrier" + actual = self.process.stdout.readline().strip() + assert actual == expected, (actual, expected, self.process.poll()) + + def command(self, value: str) -> None: + assert self.process.stdin is not None + self.process.stdin.write(value + "\n") + self.process.stdin.flush() + + +@contextmanager +def peer_case(tmp_path: Path, mode: str = "valid", *, namespace: bool = False) -> Iterator[PeerCase]: + paths = {name: str(tmp_path / name) for name in ("original", "target", "controller", "other")} + descriptors: dict[str, int] = {} + process = None + connection = None + mailbox, child_mailbox = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + mailbox.settimeout(10) + try: + for name, path in paths.items(): + descriptors[name] = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + fcntl.flock(descriptors["original"], fcntl.LOCK_SH | fcntl.LOCK_NB) + fcntl.flock(descriptors["controller"], fcntl.LOCK_EX | fcntl.LOCK_NB) + inherited: dict[str, int] = {} + if mode == "inherited-original": + inherited["original"] = descriptors["original"] + elif mode == "inherited-target": + fcntl.flock(descriptors["target"], fcntl.LOCK_EX | fcntl.LOCK_NB) + inherited["target"] = descriptors["target"] + process = subprocess.Popen( + [sys.executable, "-I", "-B", "-c", _CONSUMER, str(child_mailbox.fileno()), json.dumps(paths), mode, json.dumps(inherited)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + pass_fds=(*inherited.values(), child_mailbox.fileno()), + ) + child_mailbox.close() + # The independent consumer creates the session pair, so SO_PEERCRED is + # its real PID/UID. The inherited pair only transports the issuer end. + # This needs no listening socket, proc substitution or credential mock. + payload, ancillary, flags, _ = mailbox.recvmsg(1, socket.CMSG_SPACE(array.array("i").itemsize), socket.MSG_CMSG_CLOEXEC) + assert payload == b"p" and not flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC) and len(ancillary) == 1 + level, kind, data = ancillary[0] + assert (level, kind) == (socket.SOL_SOCKET, socket.SCM_RIGHTS) + received = array.array("i") + received.frombytes(data) + assert len(received) == 1 + connection = socket.socket(fileno=received[0]) + start = Path(f"/proc/{process.pid}/stat").read_text().rsplit(")", 1)[1].split()[19] + expected = ExpectedPeer(process.pid, start, Path("/proc/sys/kernel/random/boot_id").read_text().strip(), os.geteuid(), "request-one", "authorization-one", "target-one") + fences = tuple(HeldFence(name, descriptors[name], kind) for name, kind in (("original", "original-sh"), ("target", "target-ex"), ("controller", "transferred-ex"))) + resources = () + if namespace: + descriptors["namespace"] = os.open("/proc/self/ns/net", os.O_RDONLY | os.O_CLOEXEC) + resources = (HeldResource("namespace", descriptors["namespace"]),) + case = PeerCase(process, connection, expected, fences, resources, paths) + case.line("challenge-sent") + yield case + finally: + if connection is not None: + connection.close() + mailbox.close() + child_mailbox.close() + if process is not None: + if process.poll() is None: + try: + assert process.stdin is not None + process.stdin.write("exit\n") + process.stdin.flush() + except BrokenPipeError: + pass + try: + process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5) + for descriptor in descriptors.values(): + os.close(descriptor) + + +def receive_custody(case: PeerCase, session: PeerSession) -> PeerCustody: + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(session.offer_and_verify_custody, case.fences, case.resources) + case.line("offer-received") + assert not future.done(), "received descriptors alone must not establish acknowledged custody" + case.command("receipt") + case.line("receipt-sent") + return future.result(timeout=10) + + +@pytest.mark.parametrize("namespace", [False, True]) +def test_real_peer_independently_owns_fences_and_receives_exclusive_custody(tmp_path: Path, namespace: bool) -> None: + with peer_case(tmp_path, namespace=namespace) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + assert custody.peer == case.expected + assert len(custody.challenge) == len(custody.receipt_nonce) == 64 + assert len(custody.resources) == 3 + int(namespace) + original, target, controller = custody.resources[:3] + assert original.peer_lock is not None and original.peer_lock.pid == case.process.pid and original.peer_lock.mode == "READ" + assert original.identity.lock is not None and original.identity.lock.pid == os.getpid() + assert target.peer_lock is not None and target.peer_lock.pid == case.process.pid and target.peer_lock.mode == "WRITE" + assert controller.peer_lock == controller.identity.lock + assert controller.peer_lock is not None and controller.peer_lock.pid == os.getpid() + if namespace: + assert custody.resources[-1].identity.kind == "network-namespace" and custody.resources[-1].peer_lock is None + session.check(custody) + assert not hasattr(session, "send_grant") + + +@pytest.mark.parametrize("mode,barrier", [("queued-only", "offer-queued"), ("valid", "offer-received")]) +def test_queued_or_received_descriptors_without_acknowledgment_never_complete(tmp_path: Path, mode: str, barrier: str) -> None: + with peer_case(tmp_path, mode) as case, PeerSession(case.connection, case.expected, timeout=0.5) as session: + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(session.offer_and_verify_custody, case.fences) + case.line(barrier) + assert not future.done(), "the producer must wait for the actual challenge-bound receipt" + with pytest.raises(PeerUnavailable, match="could not be verified"): + future.result(timeout=5) + assert case.connection.fileno() == -1 + for fence in case.fences: + os.fstat(fence.descriptor) + + +@pytest.mark.parametrize( + "mode", + ["omitted", "duplicate-fd", "duplicate-name", "unknown-name", "stale-nonce", "wrong-fd", "reopened-controller", "closed-controller", "boolean-fd", "rights-receipt", "inherited-original", "inherited-target", "unlocked-original"], +) +def test_actual_bad_or_unheld_custody_refuses(tmp_path: Path, mode: str) -> None: + with peer_case(tmp_path, mode) as case, PeerSession(case.connection, case.expected) as session: + with pytest.raises(PeerUnavailable): + receive_custody(case, session) + assert case.connection.fileno() == -1 + + +@pytest.mark.parametrize("mode", ["wrong-target", "bad-challenge", "oversized-challenge", "duplicate-challenge", "rights-challenge"]) +def test_challenge_is_bounded_exact_and_cannot_supply_descriptors(tmp_path: Path, mode: str) -> None: + with peer_case(tmp_path, mode) as case: + before = len(tuple(Path("/proc/self/fd").iterdir())) + with pytest.raises((PeerUnavailable, ValueError)): + PeerSession(case.connection, case.expected) + assert case.connection.fileno() == -1 + assert len(tuple(Path("/proc/self/fd").iterdir())) <= before - 1 + + +@pytest.mark.parametrize("change", ["pid", "uid", "start_token", "boot_id"]) +def test_socket_credentials_and_current_process_lifetime_are_independent(tmp_path: Path, change: str) -> None: + with peer_case(tmp_path) as case: + values = {"pid": os.getpid(), "uid": os.geteuid() + 1, "start_token": str(int(case.expected.start_token) + 1), "boot_id": "00000000-0000-4000-8000-000000000001"} + with pytest.raises(PeerUnavailable): + PeerSession(case.connection, replace(case.expected, **{change: values[change]})) + assert case.connection.fileno() == -1 + + +@pytest.mark.parametrize("command", ["close-original", "unlock-original", "unlock-target", "close-controller", "unlock-controller", "close-namespace"]) +def test_custody_is_rechecked_after_receipt(tmp_path: Path, command: str) -> None: + with peer_case(tmp_path, namespace=True) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + case.command(command) + case.line(command) + with pytest.raises(PeerUnavailable): + session.check(custody) + assert case.connection.fileno() == -1 + + +def test_exited_peer_never_remains_current(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + case.command("exit") + case.process.wait(timeout=5) + with pytest.raises(PeerUnavailable): + session.check(custody) + + +@pytest.mark.parametrize("change", ["replacement", "missing", "protection", "host-unlock"]) +def test_changed_issuer_fence_refuses_after_receipt(tmp_path: Path, change: str) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + path = Path(case.paths["original"]) + if change in ("replacement", "missing"): + path.rename(path.with_name("preserved-original")) + if change == "replacement": + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + elif change == "protection": + path.chmod(0o644) + else: + fcntl.flock(case.fences[0].descriptor, fcntl.LOCK_UN) + with pytest.raises(PeerUnavailable): + session.check(custody) + + +def test_duplicate_held_resource_cannot_be_disguised_with_another_name(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + duplicate = replace(case.fences[0], name="another-original") + with pytest.raises(PeerUnavailable, match="duplicated"): + session.offer_and_verify_custody((*case.fences, duplicate)) + + +def test_custody_from_a_different_session_is_not_accepted(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + custody = receive_custody(case, session) + with pytest.raises(PeerUnavailable, match="another session"): + session.check(replace(custody, receipt_nonce="0" * 64)) + + +def test_session_can_offer_only_once(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + receive_custody(case, session) + with pytest.raises(PeerUnavailable, match="already offered"): + session.offer_and_verify_custody(case.fences) + + +def test_regular_file_cannot_be_claimed_as_namespace_resource(tmp_path: Path) -> None: + with peer_case(tmp_path) as case: + with pytest.raises(PeerUnavailable, match="network namespace"): + inspect_held_resource(HeldResource("pretend-namespace", case.fences[0].descriptor)) + + +def test_partial_receipt_does_not_extend_the_total_exchange_deadline(tmp_path: Path) -> None: + with peer_case(tmp_path, "drip-receipt") as case, PeerSession(case.connection, case.expected, timeout=0.2) as session: + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(session.offer_and_verify_custody, case.fences) + case.line("offer-received") + case.command("receipt") + with pytest.raises(PeerUnavailable): + future.result(timeout=2) + + +def test_peer_exit_before_receipt_never_establishes_custody(tmp_path: Path) -> None: + with peer_case(tmp_path) as case, PeerSession(case.connection, case.expected) as session: + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit(session.offer_and_verify_custody, case.fences) + case.line("offer-received") + case.command("exit") + case.process.wait(timeout=5) + with pytest.raises(PeerUnavailable): + future.result(timeout=2) From 355143f5da4936b1c628fcb22a932581e2fcd59b Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 01:14:42 +0900 Subject: [PATCH 04/24] test(management): exercise native recovery refusals and publication boundaries --- .../workflows/management-recovery-epoch.yml | 14 +- scripts/recovery_epoch_fault_tracer.py | 23 +- scripts/recovery_epoch_native_negatives.py | 510 ++++++++++++++++++ scripts/recovery_vm/README.md | 66 +++ scripts/recovery_vm/compose.kvm.yaml | 6 + scripts/recovery_vm/compose.yaml | 4 - scripts/recovery_vm/guest.sh | 9 +- scripts/recovery_vm/prepare.sh | 27 +- scripts/recovery_vm/runner.sh | 8 +- .../test_management_recovery_epoch_host.py | 217 +++++++- tests/test_management_recovery_epoch_host.py | 39 +- ...agement_recovery_epoch_native_negatives.py | 278 ++++++++++ 12 files changed, 1165 insertions(+), 36 deletions(-) create mode 100644 scripts/recovery_epoch_native_negatives.py create mode 100644 scripts/recovery_vm/README.md create mode 100644 scripts/recovery_vm/compose.kvm.yaml create mode 100644 tests/test_management_recovery_epoch_native_negatives.py diff --git a/.github/workflows/management-recovery-epoch.yml b/.github/workflows/management-recovery-epoch.yml index 01e15ec41..636972336 100644 --- a/.github/workflows/management-recovery-epoch.yml +++ b/.github/workflows/management-recovery-epoch.yml @@ -42,12 +42,14 @@ jobs: version: "0.13.6" - name: Install locked test dependencies run: uv sync --locked --dev --extra test - - name: Require actual hardware virtualization + - name: Inspect the actual guest acceleration device run: | set -euo pipefail - test -c /dev/kvm - test -r /dev/kvm - test -w /dev/kvm + if test -c /dev/kvm; then + stat --format='KVM device: %n mode=%a uid=%u gid=%g' /dev/kvm + else + echo 'No KVM device: the fresh QEMU guest will use bounded software CPU emulation.' + fi - name: Build and verify exact installed management image run: uv run python scripts/prepare_management_test_image.py - name: Prepare the approved fresh recovery VM @@ -64,12 +66,12 @@ jobs: caddy_id="$(docker image inspect --format '{{.Id}}' caddy:2)" bash scripts/recovery_vm/prepare.sh "$RECOVERY_FIXTURE" "$APPROVED_AUTHORIZATION_ID" "$manager_id" "$caddy_id" "$token_file" "$RECOVERY_SCENARIO" - name: Build bounded QEMU runner - run: docker compose --env-file "$RECOVERY_FIXTURE/compose.env" -f scripts/recovery_vm/compose.yaml build + run: docker compose --env-file "$RECOVERY_FIXTURE/compose.env" build - name: Run actual original producers, containment and one-shot issuer timeout-minutes: 125 run: | set -euo pipefail - docker compose --env-file "$RECOVERY_FIXTURE/compose.env" -f scripts/recovery_vm/compose.yaml run --rm fixture + docker compose --env-file "$RECOVERY_FIXTURE/compose.env" run --rm fixture - name: Remove the ephemeral read credential if: always() run: | diff --git a/scripts/recovery_epoch_fault_tracer.py b/scripts/recovery_epoch_fault_tracer.py index a63510597..f843cb17d 100644 --- a/scripts/recovery_epoch_fault_tracer.py +++ b/scripts/recovery_epoch_fault_tracer.py @@ -14,10 +14,10 @@ import os import platform import signal -import subprocess import threading from dataclasses import dataclass from pathlib import Path +from typing import Protocol from auto_coder.logger_config import get_logger @@ -82,6 +82,15 @@ class PublicationObservation: directory_fsync_observed: bool +class TraceProcess(Protocol): + """Actual process identity with a caller-owned termination observer.""" + + pid: int + returncode: int | None + + def wait(self, timeout: float | None = None) -> int: ... + + class KernelPublicationBarrier: """Attach to one actual child and retain its exact syscall stop. @@ -92,7 +101,7 @@ class KernelPublicationBarrier: kill or release finishes tracing; close kills an unfinished tracee. """ - def __init__(self, issuer: subprocess.Popen, *, root: Path, name: str, when: str): + def __init__(self, issuer: TraceProcess, *, root: Path, name: str, when: str): if platform.system() != "Linux" or platform.machine() != "x86_64": raise ValueError("the native publication fixture requires Linux x86_64") if name not in PUBLICATIONS or when not in {"before", "after"}: @@ -114,7 +123,7 @@ def __init__(self, issuer: subprocess.Popen, *, root: Path, name: str, when: str self._done = threading.Event() self._decision = "kill" self._error: BaseException | None = None - self._observation: PublicationObservation | None = None + self._observation: object | None = None self._thread: threading.Thread | None = None self._file_identity: tuple[int, int] | None = None self._file_synced = False @@ -284,7 +293,7 @@ def resume_initial(self) -> None: raise ValueError("no active initial publication barrier") self._resume.set() - def wait(self, timeout: float = 900) -> PublicationObservation: + def wait_observation(self, timeout: float = 900) -> object: if not self._hit.wait(timeout): self.close() raise TimeoutError("the actual issuer never reached the selected publication") @@ -293,6 +302,12 @@ def wait(self, timeout: float = 900) -> PublicationObservation: raise AssertionError("the native publication observation is missing") return self._observation + def wait(self, timeout: float = 900) -> PublicationObservation: + observation = self.wait_observation(timeout) + if not isinstance(observation, PublicationObservation): + raise AssertionError("the native stop is not a publication observation") + return observation + def _finish(self, action: str) -> None: self._raise_error() if self._observation is None or self._done.is_set(): diff --git a/scripts/recovery_epoch_native_negatives.py b/scripts/recovery_epoch_native_negatives.py new file mode 100644 index 000000000..ef7d26687 --- /dev/null +++ b/scripts/recovery_epoch_native_negatives.py @@ -0,0 +1,510 @@ +"""Actual refusal injections for the separately authorized recovery epoch VM. + +The calling driver supplies only process orchestration and a preservation +projection. Every authority decision still comes from the original production +issuer or the separate read-only containment CLI. No injection runs after an +original epoch publication, and an unexpected publication is never repaired. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import re +import select +import shutil +import signal +import socket +import stat +import subprocess +import sys +import time +import uuid +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable, ContextManager, Iterator + +from scripts.test_management_containment_closed_host import MARKER, read_authorization +from scripts.test_management_containment_enrollment_host import _docker, _systemctl, _write_private + +from auto_coder.logger_config import get_logger +from auto_coder.management.containment_kernel_subobservation import _holder_start_token, _require_capture_host +from auto_coder.management.containment_route_manifest import _systemd_catalog, _systemd_unit +from auto_coder.management.fresh_host import _evidence +from auto_coder.management.fresh_host_observation import _engine_id, assert_host_listener_absent +from auto_coder.management.installed_bootstrap_mount_profile import _inspect +from auto_coder.management.original_attempts import canonical +from auto_coder.management.recovery_containment_model import RECOVERY_ENROLLMENT_ROOT, NamedDescriptor, RecoveryEnrollment, RecoveryLock, RecoveryOriginal, RecoveryResources +from auto_coder.management.recovery_containment_profile import digest +from auto_coder.management.recovery_epoch_peer import inspect_kernel_lock +from auto_coder.management.recovery_epoch_state import EPOCH_ROOT + +logger = get_logger(__name__) +_FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") +_UNRESERVED_ENTRIES = frozenset({"epoch.lock", "inventory.json"}) +REQUIRED_NEGATIVE_CASES = frozenset( + { + "busy-original-owner-refused", + "busy-controller-fence-refused", + "replaced-original-owner-refused", + "replaced-epoch-lock-refused", + "unknown-docker-management-route-refused", + "omitted-installed-controller-refused", + "unmasked-installed-restarter-refused", + "retained-old-network-task-and-listener-refused", + "old-published-listener-refused", + } +) +REQUIRED_REPORT_CASES = frozenset( + { + "native-current-containment-reverified", + "forged-native-containment-refused", + "cross-deployment-containment-refused", + "stale-native-containment-refused", + } +) + + +@dataclass(frozen=True) +class NativeNegativeContext: + enrollment: RecoveryEnrollment + fixture_root: Path + epoch_root: Path + + +class EpochAlreadyPublished(AssertionError): + """The fresh fixture has become uncertain and must never be reused.""" + + +def assert_unreserved_epoch(root: Path) -> None: + """Reject every record, head, partial name or unexpected epoch object.""" + if not root.is_absolute() or root != root.resolve(strict=True): + raise EpochAlreadyPublished("the fixed epoch directory is no longer canonical; never repair or retry") + directory = root.lstat() + if not stat.S_ISDIR(directory.st_mode) or directory.st_uid != os.geteuid() or stat.S_IMODE(directory.st_mode) != 0o700 or {path.name for path in root.iterdir()} != _UNRESERVED_ENTRIES: + raise EpochAlreadyPublished("the epoch has an original publication, partial state or changed root; never repair or retry") + for name in _UNRESERVED_ENTRIES: + info = (root / name).lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1: + raise EpochAlreadyPublished("an original epoch input is missing or replaced by an unsupported object; never repair or retry") + + +def _projection(path: Path) -> tuple[object, ...]: + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise AssertionError("the fixture selected a nonregular original") + return (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_nlink, info.st_size, info.st_mtime_ns, hashlib.sha256(path.read_bytes()).hexdigest()) + + +def _epoch_snapshot(root: Path) -> str: + assert_unreserved_epoch(root) + info = root.lstat() + return digest(((info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid), tuple((name, _projection(root / name)) for name in sorted(_UNRESERVED_ENTRIES)))) + + +def require_native_context(context: NativeNegativeContext, *, unreserved: bool = True) -> None: + """Verify the original VM approval before a separately bounded scenario. + + Late interruption fixtures select ``unreserved=False`` only when they also + verify their exact original publication prefix; it never permits a reset. + """ + if context.epoch_root != EPOCH_ROOT or context.fixture_root == _FIXTURE_ROOT or not context.fixture_root.is_relative_to(_FIXTURE_ROOT) or context.fixture_root != context.fixture_root.resolve(strict=True): + raise PermissionError("native refusal injections require the fixed disposable fixture roots") + _require_capture_host() + authority = read_authorization(MARKER) + selected = context.enrollment + if ( + authority["authorization_id"] != selected.authorization_id + or authority["boot_id"] != Path("/proc/sys/kernel/random/boot_id").read_text().strip() + or authority["machine_id"] != Path("/etc/machine-id").read_text().strip() + or authority["engine_id"] != selected.engine_id + or authority["engine_id"] != _engine_id() + ): + raise PermissionError("the original separate VM authorization no longer selects this fixture") + RecoveryEnrollment.parse(json.loads(json.dumps(asdict(selected)))) + if not any(original.state == "started" for original in selected.originals) or not selected.watchers: + raise AssertionError("native negatives require actual original and controller resources") + for lock in (*(original.owner for original in selected.originals if original.owner is not None), *(watcher.lock for watcher in selected.watchers), selected.target.owner): + if not Path(lock.path).is_relative_to(context.fixture_root): + raise PermissionError("a selected negative resource is outside the authorized fixture") + if unreserved: + assert_unreserved_epoch(context.epoch_root) + + +@contextmanager +def _replacement(epoch_root: Path, path: Path, backup: Path, contents: bytes) -> Iterator[None]: + """Retain the original inode; restore only our still-unchanged injection.""" + assert_unreserved_epoch(epoch_root) + original = _projection(path) + if path != path.resolve(strict=True) or backup.exists() or backup.is_symlink() or backup.parent.stat().st_dev != path.parent.stat().st_dev: + raise AssertionError("fixture replacement requires a unique same-filesystem backup") + path.rename(backup) + replacement: tuple[object, ...] | None = None + try: + _write_private(path, contents) + replacement = _projection(path) + if replacement[:2] == original[:2]: + raise AssertionError("fixture replacement did not select a different inode") + yield + finally: + # A publication makes even an epoch-lock restoration an impermissible + # retry/reset. Keep all original and injected objects for diagnosis. + assert_unreserved_epoch(epoch_root) + if _projection(backup) != original: + raise AssertionError("the retained original changed during the negative fixture") + if replacement is not None: + if _projection(path) != replacement: + raise AssertionError("the replacement changed; refusing fixture cleanup") + path.unlink() + elif path.exists() or path.is_symlink(): + raise AssertionError("an incomplete replacement is retained for diagnosis") + backup.rename(path) + if _projection(path) != original: + raise AssertionError("fixture cleanup did not restore the exact original object") + + +@contextmanager +def _busy_fence(selected: RecoveryLock) -> Iterator[None]: + descriptor = os.open(selected.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino) != (selected.device, selected.inode) or not stat.S_ISREG(info.st_mode): + raise AssertionError("fixture contention did not open the selected original") + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + lock = inspect_kernel_lock(os.getpid(), descriptor, selected.device, selected.inode) + if lock is None or lock.mode != "WRITE" or lock.pid != os.getpid(): + raise AssertionError("fixture contender lacks its real independently acquired EX") + yield + if inspect_kernel_lock(os.getpid(), descriptor, selected.device, selected.inode) != lock: + raise AssertionError("the actual fixture contention was released during refusal") + finally: + os.close(descriptor) + + +@contextmanager +def _live_process(command: tuple[str, ...], *, pass_fds: tuple[int, ...] = ()) -> Iterator[subprocess.Popen[bytes]]: + """Bound a real injection process to its original pidfd and ready message.""" + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=pass_fds) + lifetime = -1 + try: + lifetime = os.pidfd_open(process.pid, 0) + if process.stdout is None: + raise AssertionError("fixture process has no readiness pipe") + message = b"" + deadline = time.monotonic() + 15 + while b"\n" not in message and len(message) <= 64: + left = deadline - time.monotonic() + if left <= 0 or not select.select((process.stdout,), (), (), left)[0]: + raise AssertionError("actual negative process never reached its bounded readiness barrier") + chunk = os.read(process.stdout.fileno(), 65 - len(message)) + if not chunk: + raise AssertionError("actual negative process exited before readiness") + message += chunk + if message != b"READY\n" or process.poll() is not None: + raise AssertionError("actual negative process did not report exact readiness") + yield process + if process.poll() is not None: + raise AssertionError("the actual injected process exited during refusal") + finally: + if process.poll() is None: + if lifetime >= 0: + signal.pidfd_send_signal(lifetime, signal.SIGTERM) + else: + process.terminate() + try: + process.communicate(timeout=10) + except subprocess.TimeoutExpired: + if lifetime >= 0: + signal.pidfd_send_signal(lifetime, signal.SIGKILL) + else: + process.kill() + process.communicate(timeout=10) + if lifetime >= 0: + os.close(lifetime) + + +_NAMESPACE_LISTENER = """ +import os +import socket +import sys +os.setns(int(sys.argv[1]), os.CLONE_NEWNET) +os.close(int(sys.argv[1])) +listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +listener.bind(('127.0.0.1', 8080)) +listener.listen(1) +print('READY', flush=True) +sys.stdin.buffer.read(1) +""" + + +@contextmanager +def _retained_namespace_listener(original: RecoveryOriginal) -> Iterator[None]: + pin = original.namespace_pin + if pin is None or original.prior_evidence is None: + raise AssertionError("a genuine retained original namespace is required") + if _holder_start_token(pin.holder_pid) != pin.holder_start_token: + raise AssertionError("the original namespace holder changed before injection") + descriptor = os.open(f"/proc/{pin.holder_pid}/fd/{pin.holder_fd}", os.O_RDONLY | os.O_CLOEXEC) + try: + info = os.fstat(descriptor) + if (info.st_dev, info.st_ino) != (pin.device, pin.inode) or os.readlink(f"/proc/self/fd/{descriptor}") != pin.namespace: + raise AssertionError("the injected process did not receive the exact retained namespace") + with _live_process((sys.executable, "-I", "-B", "-c", _NAMESPACE_LISTENER, str(descriptor)), pass_fds=(descriptor,)) as process: + proc = Path("/proc") / str(process.pid) + prior, _, _ = _evidence(Path(original.prior_evidence.path)) + if os.readlink(proc / "ns/net") != pin.namespace or "0::" + prior.cgroup_path in (proc / "cgroup").read_text().splitlines(): + raise AssertionError("the real task did not remain outside the old container in its retained network") + rows = (proc / "net/tcp").read_text().splitlines()[1:] + if not any(row.split()[1].endswith(":1F90") and row.split()[3] == "0A" for row in rows): + raise AssertionError("the actual retained namespace has no injected listening socket") + yield + finally: + os.close(descriptor) + + +@contextmanager +def _published_listener(original: RecoveryOriginal, target_port: int) -> Iterator[None]: + if original.prior_evidence is None: + raise AssertionError("the old listener requires actual original capture") + prior, _, _ = _evidence(Path(original.prior_evidence.path)) + if prior.published_port == target_port: + raise AssertionError("the fixture target reused an original published port") + assert_host_listener_absent(prior.published_port) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", prior.published_port)) + listener.listen(1) + try: + assert_host_listener_absent(prior.published_port) + except ValueError: + pass + else: + raise AssertionError("the actual host listener injection was not visible") + yield + assert_host_listener_absent(prior.published_port) + + +@contextmanager +def _extra_docker(enrollment: RecoveryEnrollment) -> Iterator[None]: + image = enrollment.originals[0].management_image_id + identifier = _docker("create", "--name", "ac-recovery-unknown-" + uuid.uuid4().hex, "--restart=no", "--network=none", "--read-only", "--entrypoint", "python", image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json") + if re.fullmatch(r"[0-9a-f]{64}", identifier) is None: + raise AssertionError("the actual extra Docker object has no complete identity") + try: + observed = _inspect(identifier) + state = observed.get("State") + if observed.get("Image") != image or not isinstance(state, dict) or state.get("Status") != "created" or state.get("Pid") != 0 or state.get("Running") is not False or observed.get("RestartCount") != 0: + raise AssertionError("the actual unknown manager route is not a pristine created object") + yield + if _inspect(identifier) != observed: + raise AssertionError("the issuer changed the injected Docker object") + finally: + current = _inspect(identifier) + state = current.get("State") + if not isinstance(state, dict) or state.get("Status") != "created" or state.get("Pid") != 0 or state.get("Running") is not False or current.get("Image") != image: + raise AssertionError("the injected object acquired a lifetime; refusing destructive cleanup") + _docker("rm", identifier) + + +@contextmanager +def _omitted_controller(context: NativeNegativeContext, backups: Path) -> Iterator[None]: + path = RECOVERY_ENROLLMENT_ROOT / (context.enrollment.enrollment_id + ".routes.json") + previous = json.loads(path.read_bytes()) + routes = previous.get("systemd_routes") + selected = context.enrollment.watchers[0].unit + if not isinstance(routes, list) or sum(isinstance(route, dict) and route.get("unit") == selected for route in routes) != 1: + raise AssertionError("the approved full manifest does not contain the actual selected watcher") + previous["systemd_routes"] = [route for route in routes if route["unit"] != selected] + with _replacement(context.epoch_root, path, backups / "approved-routes.original", canonical(previous)): + yield + + +@contextmanager +def _unmasked_restarter(context: NativeNegativeContext) -> Iterator[None]: + """Add an actual runnable restarter without touching original mask inodes.""" + catalog = _systemd_catalog() + executable = shutil.which("docker") + if executable is None: + raise AssertionError("the original local Docker executable is unavailable") + executable = str(Path(executable).resolve(strict=True)) + if any(character in executable for character in ' \t\n\r%"\\'): + raise AssertionError("the native Docker executable is outside the finite systemd profile") + name = "auto-coder-recovery-negative-" + uuid.uuid4().hex + ".service" + path = Path("/run/systemd/system") / name + contents = f"[Unit]\nDescription=Disposable management recovery restarter negative\n[Service]\nType=oneshot\nExecStart={executable} start {context.enrollment.originals[0].management_id}\n".encode() + _write_private(path, contents) + identity = _projection(path) + try: + _systemctl("daemon-reload") + actual = _systemd_unit(name) + if actual.inhibited or not actual.management_candidate or actual.main_pid != 0 or actual.active_state != "inactive" or name not in _systemd_catalog(): + raise AssertionError("the actual installed restarter is not the intended unmasked inactive route") + yield + finally: + if _projection(path) != identity: + raise AssertionError("the fixture restarter source changed; refusing cleanup") + path.unlink() + _systemctl("daemon-reload") + deadline = time.monotonic() + 15 + while _systemd_catalog() != catalog: + if time.monotonic() >= deadline: + raise AssertionError("the original systemd catalog did not return after exact fixture removal") + time.sleep(0.1) + + +def _run_case(context: NativeNegativeContext, label: str, injection: Callable[[], ContextManager[None]], run_refusal: Callable[[str], None], preserved_snapshot: Callable[[], object]) -> None: + assert_unreserved_epoch(context.epoch_root) + baseline = preserved_snapshot() + epoch_before = _epoch_snapshot(context.epoch_root) + logger.info("Starting native prepublication refusal case: {}", label) + with injection(): + injected = preserved_snapshot() + epoch_injected = _epoch_snapshot(context.epoch_root) + try: + run_refusal(label) + finally: + assert_unreserved_epoch(context.epoch_root) + if _epoch_snapshot(context.epoch_root) != epoch_injected: + raise AssertionError("the refused issuer changed an original epoch input") + if preserved_snapshot() != injected: + raise AssertionError("the refused issuer changed original/OAuth/worker state during " + label) + if _epoch_snapshot(context.epoch_root) != epoch_before or preserved_snapshot() != baseline: + raise AssertionError("the exact injected fixture state did not restore after " + label) + logger.info("Native prepublication refusal and preservation verified: {}", label) + + +def run_native_negatives(context: NativeNegativeContext, run_refusal: Callable[[str], None], preserved_snapshot: Callable[[], object]) -> list[str]: + """Exercise actual issuer refusals before its first irreversible publication.""" + require_native_context(context) + backups = context.fixture_root / "native-negative-originals" + backups.mkdir(mode=0o700) + original = next(item for item in context.enrollment.originals if item.state == "started") + if original.owner is None: + raise AssertionError("the original started owner identity is absent") + selected_owner = original.owner + owner = Path(selected_owner.path) + epoch = context.epoch_root / "epoch.lock" + injections: tuple[tuple[str, Callable[[], ContextManager[None]]], ...] = ( + ("busy-original-owner-refused", lambda: _busy_fence(selected_owner)), + ("busy-controller-fence-refused", lambda: _busy_fence(context.enrollment.watchers[0].lock)), + ("replaced-original-owner-refused", lambda: _replacement(context.epoch_root, owner, backups / "owner.original", owner.read_bytes())), + ("replaced-epoch-lock-refused", lambda: _replacement(context.epoch_root, epoch, backups / "epoch.original", epoch.read_bytes())), + ("unknown-docker-management-route-refused", lambda: _extra_docker(context.enrollment)), + ("omitted-installed-controller-refused", lambda: _omitted_controller(context, backups)), + ("unmasked-installed-restarter-refused", lambda: _unmasked_restarter(context)), + ("retained-old-network-task-and-listener-refused", lambda: _retained_namespace_listener(original)), + ("old-published-listener-refused", lambda: _published_listener(original, context.enrollment.target.published_port)), + ) + cases = [] + for label, injection in injections: + _run_case(context, label, injection, run_refusal, preserved_snapshot) + cases.append(label) + if set(cases) != REQUIRED_NEGATIVE_CASES: + raise AssertionError("the native refusal matrix is incomplete") + return cases + + +def _issuer_resources(enrollment: RecoveryEnrollment, issuer_pid: int) -> RecoveryResources: + """Read actual stopped issuer descriptors; the production verifier rechecks.""" + if type(issuer_pid) is not int or issuer_pid <= 1: + raise AssertionError("an exact actual issuer process is required") + proc = Path("/proc") / str(issuer_pid) + fields = (proc / "stat").read_text().rsplit(")", 1)[1].split() + if fields[0] not in {"T", "t"}: + raise AssertionError("native report probes require the actual prepublication SIGSTOP/ptrace barrier") + start = _holder_start_token(issuer_pid) + descriptors = tuple((proc / "fd").iterdir()) + if len(descriptors) > 4096 or any(not item.name.isdecimal() for item in descriptors): + raise AssertionError("the actual issuer descriptor inventory exceeds the fixture bound") + + def find(name: str, path: str, device: int, inode: int) -> NamedDescriptor: + matches = [] + for candidate in descriptors: + info = candidate.stat() + if (info.st_dev, info.st_ino) == (device, inode) and os.readlink(candidate) == path: + matches.append(int(candidate.name)) + if len(matches) != 1: + raise AssertionError("the actual issuer resource is absent or ambiguous: " + name) + return NamedDescriptor(name, matches[0]) + + owners = tuple(find(item.generation_id, item.owner.path, item.owner.device, item.owner.inode) for item in enrollment.originals if item.owner is not None) + controllers = tuple(find(item.unit, item.lock.path, item.lock.device, item.lock.inode) for item in enrollment.watchers) + namespaces = tuple(find(item.generation_id, item.namespace_pin.namespace, item.namespace_pin.device, item.namespace_pin.inode) for item in enrollment.originals if item.namespace_pin is not None) + return RecoveryResources(issuer_pid, start, Path("/proc/sys/kernel/random/boot_id").read_text().strip(), owners, controllers, namespaces) + + +def _containment_cli(directory: Path, label: str, enrollment: str, resources: Path, *, previous: Path | None = None, refusal: str | None = None) -> dict[str, object]: + command = [sys.executable, "-B", "-m", "auto_coder.management.recovery_containment", "verify" if previous is not None else "observe", "--enrollment", enrollment, "--resources", str(resources)] + if previous is not None: + command.extend(("--previous", str(previous))) + completed = subprocess.run(command, capture_output=True, text=True, timeout=300, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + _write_private(directory / (label + ".log"), (completed.stdout + completed.stderr).encode()) + value = json.loads(completed.stdout.splitlines()[-1]) + if not isinstance(value, dict) or any(value.get(name) is not False for name in ("grant_issued", "history_asserted", "admission_created")): + raise AssertionError("the actual containment CLI returned an incomplete or authorizing result") + if refusal is None: + if completed.returncode != 0 or value.get("status") != "observed-current-recovery-containment": + raise AssertionError("the genuine native containment positive control was unavailable: " + str(value.get("reason"))) + elif completed.returncode != 2 or value.get("status") != "unavailable" or refusal not in str(value.get("reason")): + raise AssertionError("the actual containment verifier did not reject the injected condition: " + label + " " + str(value.get("reason"))) + return value + + +def verify_native_containment_reports(context: NativeNegativeContext, issuer_pid: int) -> list[str]: + """Reobserve saved/forged reports under genuine paused issuer custody.""" + require_native_context(context) + directory = context.fixture_root / "native-containment-report-probes" + directory.mkdir(mode=0o700) + lifetime = os.pidfd_open(issuer_pid, 0) + try: + epoch = _epoch_snapshot(context.epoch_root) + resources = _issuer_resources(context.enrollment, issuer_pid) + resource_path = directory / "actual-issuer-resources.json" + _write_private(resource_path, canonical(asdict(resources))) + enrollment = context.enrollment.enrollment_id + poller = select.poll() + poller.register(lifetime, select.POLLIN) + + def check_barrier() -> None: + if poller.poll(0) or _epoch_snapshot(context.epoch_root) != epoch or _issuer_resources(context.enrollment, issuer_pid) != resources: + raise AssertionError("the actual issuer advanced or lost custody during native report probes") + + def probe(label: str, *, previous: Path | None = None, refusal: str | None = None) -> dict[str, object]: + check_barrier() + try: + return _containment_cli(directory, label, enrollment, resource_path, previous=previous, refusal=refusal) + finally: + check_barrier() + + try: + report = probe("original-observer") + previous = directory / "original-observation.json" + _write_private(previous, canonical(report)) + probe("separate-current-verifier", previous=previous) + cases = ["native-current-containment-reverified"] + for label, field in (("forged-native-containment-refused", "target"), ("cross-deployment-containment-refused", "deployment")): + forged = json.loads(json.dumps(report)) + if field == "target": + forged["facts"]["target"]["pid"] += 1 + else: + forged["facts"]["enrollment"]["enrollment"]["deployment_id"] = "foreign-" + uuid.uuid4().hex + forged["evidence_sha256"] = digest(forged["facts"]) + path = directory / (label + ".json") + _write_private(path, canonical(forged)) + probe(label, previous=path, refusal="previous recovery scope/source/resources/current facts differ") + cases.append(label) + original = next(item for item in context.enrollment.originals if item.state == "started") + check_barrier() + with _retained_namespace_listener(original): + probe("stale-native-containment-refused", previous=previous, refusal="prior network namespace still has a task") + cases.append("stale-native-containment-refused") + probe("restored-current-verifier", previous=previous) + if set(cases) != REQUIRED_REPORT_CASES: + raise AssertionError("the actual report-verifier case matrix is incomplete") + return cases + finally: + check_barrier() + finally: + os.close(lifetime) diff --git a/scripts/recovery_vm/README.md b/scripts/recovery_vm/README.md new file mode 100644 index 000000000..ae8d95949 --- /dev/null +++ b/scripts/recovery_vm/README.md @@ -0,0 +1,66 @@ +# Native recovery epoch fixture + +This runner boots a fresh Ubuntu VM with its own Linux kernel, initial namespaces, +systemd, Docker engine and boot identity. It does not run the host issuer on the +outer CI runner or use a privileged Docker container as a substitute for the VM. +The driver exercises original provenance, current containment, genuine installed +READY and the production issuer/consumer entrypoints. A run is acceptance only +when its exact scenario oracle succeeds and the VM exits successfully. + +Preparation selects KVM when the outer host exposes `/dev/kvm`; its separate +Compose override passes that exact device and its actual group to the unprivileged +QEMU process. The preparing shell does not need membership in the device group. +If that device is absent, preparation explicitly selects QEMU TCG CPU emulation +with the `max` CPU model. The selected accelerator is bound in the input manifest. +Both modes boot the same real kernel and execute the same native oracles. A KVM +permission or execution failure is an error; the runner does not silently switch +an already prepared KVM input to another mode. + +The outer QEMU container has a read-only root, no capabilities, no new privileges, +2 CPUs, 6 GiB memory and a 256-process limit. Only prepared input, the new guest +disk and bounded output are mounted. Each scenario gets a new directory and new +32 GiB sparse guest disk; an existing disk or original epoch is never reset. + +## Reproduce a reviewed scenario + +Use a clean, committed checkout with the locked test dependencies installed. The +read credential is used only to verify the actual immutable tested-source tag and +successful GitHub test run for the installed READY. It needs repository contents +and Actions read access, not publication authority. Keep it in an existing private +file; do not put its value on the command line. + +```sh +uv sync --locked --dev --extra test +uv run python scripts/prepare_management_test_image.py +docker pull caddy:2 +export APPROVED_AUTHORIZATION_ID=reviewed-recovery-fixture-unique +RECOVERY_FIXTURE=/var/tmp/ac-recovery-vm-unique +MANAGER_IMAGE_ID=$(docker image inspect --format '{{.Id}}' auto-coder:management-fixture-$(git rev-parse --short=12 HEAD)) +CADDY_IMAGE_ID=$(docker image inspect --format '{{.Id}}' caddy:2) +READ_TOKEN_FILE=/absolute/path/to/private-read-token +bash scripts/recovery_vm/prepare.sh "$RECOVERY_FIXTURE" "$APPROVED_AUTHORIZATION_ID" "$MANAGER_IMAGE_ID" "$CADDY_IMAGE_ID" "$READ_TOKEN_FILE" grant +docker compose --env-file "$RECOVERY_FIXTURE/compose.env" build +docker compose --env-file "$RECOVERY_FIXTURE/compose.env" run --rm fixture +``` + +The prepared `compose.env` selects the exact reviewed Compose files, including the +KVM-only device override when applicable. Do not add a separate `-f` argument +that would discard this selection. The VM-only authorization marker binds the +actual new machine, boot and engine. The driver verifies it before provisioning +any management fixture. It refuses a populated engine or earlier fixture roots. + +`SCENARIOS` and `required_cases()` in +`scripts/test_management_recovery_epoch_host.py` define the finite scenario +oracle. All before/after publication cases run on independent fresh VMs. They +use real ptrace syscall stops around the actual exclusive record/head creation +and successful file/directory fsync, with a pinned SIGKILL and no syscall-result +replacement. The native process oracles also inspect real peer custody, original +store preservation, unrelated worker lifetimes and effect requests, and absence +of a replacement child/admission. Component guard tests cannot substitute for +these executions. + +The guest read credential is deleted on exit. The outer input credential must +also be deleted after the run; the CI workflow does this even on failure. +Artifacts contain the result, bounded diagnostics and guest/serial logs, not the +original OAuth stores or a reusable authority inventory. The fixture is not an +incident-recovery procedure and cannot fill missing historical evidence. diff --git a/scripts/recovery_vm/compose.kvm.yaml b/scripts/recovery_vm/compose.kvm.yaml new file mode 100644 index 000000000..f581518b3 --- /dev/null +++ b/scripts/recovery_vm/compose.kvm.yaml @@ -0,0 +1,6 @@ +services: + fixture: + group_add: + - "${RECOVERY_KVM_GID:?Set the actual KVM device group}" + devices: + - /dev/kvm:/dev/kvm:rwm diff --git a/scripts/recovery_vm/compose.yaml b/scripts/recovery_vm/compose.yaml index 4615fa828..dbe2bf4dc 100644 --- a/scripts/recovery_vm/compose.yaml +++ b/scripts/recovery_vm/compose.yaml @@ -5,8 +5,6 @@ services: environment: APPROVED_AUTHORIZATION_ID: ${APPROVED_AUTHORIZATION_ID:?Set the matching disposable fixture authorization} user: "${RECOVERY_VM_UID:?Set the preparation UID}:${RECOVERY_VM_GID:?Set the preparation GID}" - group_add: - - "${RECOVERY_KVM_GID:?Set the KVM device group}" init: true cpus: 2.0 mem_limit: 6g @@ -16,8 +14,6 @@ services: - ALL security_opt: - no-new-privileges:true - devices: - - /dev/kvm:/dev/kvm:rwm tmpfs: - /tmp:size=128m volumes: diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh index 8d08c505e..4621aa925 100644 --- a/scripts/recovery_vm/guest.sh +++ b/scripts/recovery_vm/guest.sh @@ -34,7 +34,7 @@ for name, digest in manifest["inputs_sha256"].items(): PY export DEBIAN_FRONTEND=noninteractive apt-get update -apt-get install -y --no-install-recommends ca-certificates cron curl docker.io git python3-venv +apt-get install -y --no-install-recommends ca-certificates cron curl docker.io docker-compose-v2 git python3-venv systemctl start docker python3 -m venv /opt/fixture-tools /opt/fixture-tools/bin/pip install uv==0.13.6 @@ -42,6 +42,13 @@ mkdir -m 700 /opt/recovery-fixture /opt/reviewed-source tar --extract --gzip --file "$input/source.tar.gz" --directory /opt/recovery-fixture cd /opt/recovery-fixture /opt/fixture-tools/bin/uv sync --locked --no-editable --no-dev --extra test +.venv/bin/python - <<'PY' +from pathlib import Path +import auto_coder.management.proxy_watch_cli as watcher + +Path(watcher.__file__).chmod(0o600) +PY +docker compose version install -m 600 "$input/source-token" /run/recovery-fixture-source-token mapfile -t selected < <(python3 - "$input" <<'PY' import json diff --git a/scripts/recovery_vm/prepare.sh b/scripts/recovery_vm/prepare.sh index b8058c040..654cf33cd 100644 --- a/scripts/recovery_vm/prepare.sh +++ b/scripts/recovery_vm/prepare.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail umask 077 # Export exact reviewed source/images; preparation does not launch a guest. -if [[ "$#" != 6 || ! "$1" =~ ^/var/tmp/ac-recovery-vm-[a-zA-Z0-9_-]+$ || ! "$2" =~ ^[a-zA-Z0-9_.:-]{1,128}$ || ! "$3" =~ ^sha256:[0-9a-f]{64}$ || ! "$4" =~ ^sha256:[0-9a-f]{64}$ || ! "$6" =~ ^(grant|kill-after-intent|kill-after-consume|lost-reply)$ ]]; then +if [[ "$#" != 6 || ! "$1" =~ ^/var/tmp/ac-recovery-vm-[a-zA-Z0-9_-]+$ || ! "$2" =~ ^[a-zA-Z0-9_.:-]{1,128}$ || ! "$3" =~ ^sha256:[0-9a-f]{64}$ || ! "$4" =~ ^sha256:[0-9a-f]{64}$ || ! "$6" =~ ^[a-z][a-z-]{1,63}$ ]]; then echo 'Usage: prepare.sh /var/tmp/ac-recovery-vm-UNIQUE AUTHORIZATION_ID MANAGER_IMAGE_ID CADDY_IMAGE_ID READ_TOKEN_FILE SCENARIO' >&2 exit 2 fi @@ -15,7 +15,21 @@ source_token_file=$5 fixture_scenario=$6 tooling_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) repo_root=$(git -C "$tooling_root" rev-parse --show-toplevel) -test -c /dev/kvm +cd -- "$repo_root" +"$repo_root/.venv/bin/python" - "$fixture_scenario" <<'PY' +import sys +from scripts.test_management_recovery_epoch_host import required_cases + +required_cases(sys.argv[1]) +PY +fixture_accelerator=tcg +fixture_kvm_gid=0 +fixture_compose_files="$tooling_root/compose.yaml" +if [[ -c /dev/kvm ]]; then + fixture_accelerator=kvm + fixture_kvm_gid=$(stat -c '%g' /dev/kvm) + fixture_compose_files="$fixture_compose_files:$tooling_root/compose.kvm.yaml" +fi test -f "$source_token_file" if [[ -e "$fixture_root" || -L "$fixture_root" ]]; then echo 'The recovery fixture directory must not exist.' >&2 @@ -35,7 +49,7 @@ docker image save "$manager_image" "$proxy_image" | tee "$fixture_root/input/ima cp "$tooling_root/guest.sh" "$fixture_root/input/guest.sh" install -m 600 "$source_token_file" "$fixture_root/input/source-token" fixture_commit=$(git -C "$repo_root" rev-parse HEAD) -python3 - "$fixture_root" "$fixture_authorization" "$fixture_commit" "$manager_image" "$proxy_image" "$fixture_scenario" <<'PY' +python3 - "$fixture_root" "$fixture_authorization" "$fixture_commit" "$manager_image" "$proxy_image" "$fixture_scenario" "$fixture_accelerator" <<'PY' import hashlib import json import pathlib @@ -43,7 +57,7 @@ import sys root = pathlib.Path(sys.argv[1]) inputs = root / "input" -manifest = dict(zip(("authorization_id", "commit", "manager_image", "proxy_image", "scenario"), sys.argv[2:])) +manifest = dict(zip(("authorization_id", "commit", "manager_image", "proxy_image", "scenario", "accelerator"), sys.argv[2:])) manifest["schema"] = "auto-coder-recovery-vm-input-v1" manifest["ready_source_sha"] = "e742338b3f311e6d30a22a8f5341b8072513d813" manifest["inputs_sha256"] = {} @@ -74,6 +88,7 @@ RECOVERY_VM_TOOLING=$tooling_root RECOVERY_VM_ROOT=$fixture_root RECOVERY_VM_UID=$(id -u) RECOVERY_VM_GID=$(id -g) -RECOVERY_KVM_GID=$(stat -c '%g' /dev/kvm) +RECOVERY_KVM_GID=$fixture_kvm_gid +COMPOSE_FILE=$fixture_compose_files ENV -printf 'Prepared exact recovery source %s in %s.\n' "$fixture_commit" "$fixture_root" +printf 'Prepared exact recovery source %s in %s with QEMU %s.\n' "$fixture_commit" "$fixture_root" "$fixture_accelerator" diff --git a/scripts/recovery_vm/runner.sh b/scripts/recovery_vm/runner.sh index 744cd6fb9..c05521543 100644 --- a/scripts/recovery_vm/runner.sh +++ b/scripts/recovery_vm/runner.sh @@ -12,11 +12,17 @@ if [[ -e /disk/guest.qcow2 ]]; then echo 'A used guest disk cannot be reused as fresh recovery evidence.' >&2 exit 2 fi +accelerator=$(python3 -c 'import json; print(json.load(open("/input/manifest.json"))["accelerator"])') +case "$accelerator" in + kvm) acceleration=(-accel kvm -cpu host) ;; + tcg) acceleration=(-accel tcg,thread=multi -cpu max) ;; + *) echo 'The prepared guest accelerator is unsupported.' >&2; exit 2 ;; +esac cloud-localds /disk/seed.img /input/user-data /input/meta-data qemu-img create -f qcow2 -F qcow2 -b /opt/base.img /disk/guest.qcow2 32G set +e timeout --signal=TERM --kill-after=30s 120m qemu-system-x86_64 \ - -accel kvm -cpu host -smp 2 -m 4608 -nodefaults -no-reboot \ + "${acceleration[@]}" -smp 2 -m 4608 -nodefaults -no-reboot \ -display none -serial stdio -monitor none \ -drive file=/disk/guest.qcow2,if=virtio,format=qcow2 \ -drive file=/disk/seed.img,if=virtio,format=raw,readonly=on \ diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index ad5cb98e2..0bcb78e85 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -29,6 +29,8 @@ from typing import Callable from scripts.recovery_epoch_containment_fixture import CapturedGeneration, capture_generation, enroll_recovery_scope, selected_file, start_watcher, target_from_container +from scripts.recovery_epoch_fault_tracer import BOUNDARY_SCENARIOS, PUBLICATIONS, KernelPublicationBarrier, PublicationObservation, parse_boundary +from scripts.recovery_epoch_native_negatives import REQUIRED_NEGATIVE_CASES, REQUIRED_REPORT_CASES, NativeNegativeContext, _replacement, assert_unreserved_epoch, run_native_negatives, verify_native_containment_reports from scripts.test_management_containment_closed_host import InhibitedStockRoutes, WatcherFixture, require_disposable_host from scripts.test_management_containment_enrollment_host import DisposableSystemdRoutes, _private_directory, _write_private @@ -52,7 +54,9 @@ logger = get_logger(__name__) FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") RESULT_SCHEMA = "auto-coder-native-recovery-epoch-result-v1" -SCENARIOS = ("grant", "kill-after-intent", "kill-after-consume", "lost-reply") +ORIGINAL_NEGATIVE_CASES = frozenset({"original-journal-absence-refused", "changed-installed-ready-refused", "changed-target-management-config-refused"}) +LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused"}) +SCENARIOS = ("grant", "kill-after-intent", "kill-after-consume", "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS) REQUIRED_CASES = frozenset( { "affirmative-original-genesis", @@ -62,6 +66,7 @@ "missing-original-refused", "genuine-installed-ready-in-target", "independent-issuer-contender-refused", + "reservation-barrier-contender-refused", "complete-native-epoch-granted", "old-owner-and-controller-contenders-refused", "issuer-death-ended-current-custody", @@ -76,8 +81,8 @@ def required_cases(scenario: str) -> frozenset[str]: if scenario not in SCENARIOS: raise ValueError("unsupported native recovery fixture scenario") if scenario == "grant": - return REQUIRED_CASES - return (REQUIRED_CASES - {"complete-native-epoch-granted", "old-owner-and-controller-contenders-refused", "issuer-death-ended-current-custody", "consumed-epoch-replay-refused"}) | {"native-" + scenario, "interrupted-epoch-replay-refused"} + return REQUIRED_CASES | REQUIRED_NEGATIVE_CASES | REQUIRED_REPORT_CASES | ORIGINAL_NEGATIVE_CASES | LINEAGE_REPORT_CASES + return (REQUIRED_CASES - {"complete-native-epoch-granted", "old-owner-and-controller-contenders-refused", "issuer-death-ended-current-custody", "consumed-epoch-replay-refused", "reservation-barrier-contender-refused"}) | {"native-" + scenario, "interrupted-epoch-replay-refused"} class WorkerFixture(Chain): @@ -110,7 +115,7 @@ def _digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def _snapshot(roots: tuple[Path, ...]) -> str: +def _snapshot(roots: tuple[Path, ...], *, semantic: bool = False) -> str: """Fingerprint protected original bytes/objects without exporting their data.""" entries = [] for root in roots: @@ -126,7 +131,10 @@ def _snapshot(roots: tuple[Path, ...]) -> str: content = "socket" else: raise AssertionError("unsupported original fixture object") - entries.append((str(path), info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_size, info.st_mtime_ns, info.st_ctime_ns, content)) + if semantic: + entries.append((str(path), info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_size if path.is_file() else 0, info.st_mtime_ns if path.is_file() else 0, content)) + else: + entries.append((str(path), info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_size, info.st_mtime_ns, info.st_ctime_ns, content)) return hashlib.sha256(canonical(entries)).hexdigest() @@ -245,6 +253,12 @@ def _check_lineage(chain: Chain, captures: tuple[CapturedGeneration, ...]) -> di complete = derive_lineage(chain.history_binding, history, tuple(descriptors)) assert not complete.unavailable, complete.unavailable assert verify_lineage(complete, chain.history_binding, history, tuple(descriptors)) == complete + try: + verify_lineage(replace(complete, generations=complete.generations[:-1]), chain.history_binding, history, tuple(descriptors)) + except OriginalEvidenceUnavailable: + pass + else: + raise AssertionError("a truncated original lineage was accepted as complete") outcomes = {item.instance_id: item.outcome for item in complete.generations} assert set(outcomes) == {"A", "B", "C", "D"} assert outcomes["B"] == "succeeded" and outcomes["C"] in {"launched", "unknown"} and outcomes["D"] == "never-created" @@ -325,7 +339,7 @@ def _target( crash_budget=0, management_version_root=versions, management_config=Path("/bootstrap/management.yaml"), - verification_timeout=600, + verification_timeout=1800, recovery_epoch_config=Path("/bootstrap/recovery-epoch.json"), ) runtime = runtime_for_receipt(bootstrap, receipt) @@ -340,7 +354,7 @@ def _target( "crash_budget": 0, "management_version_root": str(versions), "management_config": str(bootstrap.management_config), - "verification_timeout": 600, + "verification_timeout": 1800, "recovery_epoch_config": "/bootstrap/recovery-epoch.json", } config = root / "bootstrap" / "config.json" @@ -491,6 +505,14 @@ def validate_result(value: dict[str, object]) -> None: assert isinstance(value.get("durable"), dict) if scenario == "grant": assert set(value["durable"]["record_sha256"]) == {"reserved", "intent", "consumed", "receipt"} + elif scenario in BOUNDARY_SCENARIOS: + publication, when = parse_boundary(scenario) + assert value["durable"]["publication"] == publication and value["durable"]["when"] == when + assert value["durable"]["consumer_grant_acknowledged"] is False + assert value["durable"]["file_fsync_observed"] is (when == "after") and value["durable"]["directory_fsync_observed"] is (when == "after") + elif scenario == "retarget-after-reservation": + assert value["durable"]["phase"] == "reserved" and value["durable"]["consumer_grant_acknowledged"] is False + assert value["durable"]["inventory_sha256"] != value["durable"]["retargeted_inventory_sha256"] else: assert value["durable"]["phase"] == ("intent" if scenario == "kill-after-intent" else "consumed" if scenario == "kill-after-consume" else "receipt") assert value["durable"]["consumer_grant_acknowledged"] is False @@ -527,6 +549,39 @@ def _interrupted_issuer(scenario: str, issuer: subprocess.Popen, issuer_log: Pat return {"phase": phase, "record_sha256": hashlib.sha256(encoded).hexdigest(), "consumer_grant_acknowledged": False, "grant_send_completed": False} +def _publication_interruption(scenario: str, observed: PublicationObservation, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: + """Verify exactly the actual interrupted original publication prefix.""" + name, when = parse_boundary(scenario) + assert (observed.name, observed.when) == (name, when) + index = PUBLICATIONS.index(name) + (1 if when == "after" else 0) + assert set(os.listdir(EPOCH_ROOT)) == {"inventory.json", "epoch.lock", *PUBLICATIONS[:index]}, "issuer crossed the selected native publication boundary" + if when == "after": + published = EPOCH_ROOT / name + info = published.stat() + assert (observed.file_device, observed.file_inode, observed.file_sha256) == (info.st_dev, info.st_ino, _digest(published)) + assert observed.file_fsync_observed and observed.directory_fsync_observed + else: + assert observed.file_device is None and observed.file_inode is None and observed.file_sha256 is None + assert not observed.file_fsync_observed and not observed.directory_fsync_observed + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "native publication interruption ends target custody", timeout=90) + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() + assert "One consumed recovery epoch grant send completed" not in issuer_log.read_text() + before = _snapshot((EPOCH_ROOT,)) + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == before, "interrupted publication or original endpoint was repaired/reissued" + return { + "publication": name, + "when": when, + "file_fsync_observed": observed.file_fsync_observed, + "directory_fsync_observed": observed.directory_fsync_observed, + "consumer_grant_acknowledged": False, + "grant_send_completed": False, + "published_names": list(PUBLICATIONS[:index]), + "native_observation": asdict(observed), + } + + def _passed(result: dict[str, object], cases: list[str], preserved: str, worker_claims: str, worker_baseline: dict[str, object], worker_posts: dict[str, int]) -> None: cases.extend(("originals-oauth-and-workers-preserved", "no-consumer-child-or-admission")) result.update(status="passed", no_child_started=True, preserved_originals=True, preserved_workers=True, preserved_originals_sha256=preserved, preserved_worker_claims_sha256=worker_claims, worker_lifetimes=worker_baseline, worker_effect_posts=worker_posts) @@ -534,6 +589,96 @@ def _passed(result: dict[str, object], cases: list[str], preserved: str, worker_ logger.info("All {} native recovery ownership cases passed; no consumer child was launched", len(cases)) +def _issuer_refusal(context: NativeNegativeContext, target: PreparedTarget, label: str) -> None: + """Run the actual complete issuer, allowing no first publication on error.""" + assert_unreserved_epoch(context.epoch_root) + log_path = context.fixture_root / ("refusal-" + label + ".log") + assert Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false" + process = None + started = False + with log_path.open("xb") as output: + try: + process = subprocess.Popen([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], stdout=output, stderr=subprocess.STDOUT, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + _wait(lambda: process.poll() is not None or PEER_SOCKET.exists() and "Fixed recovery issuer endpoint is listening" in log_path.read_text(), "actual refused issuer startup " + label, timeout=180) + if process.poll() is None: + Chain.docker("start", target.generation.container) + started = True + _wait(lambda: process.poll() is not None or set(os.listdir(context.epoch_root)) != {"inventory.json", "epoch.lock"}, "actual prepublication refusal " + label, timeout=1800) + assert_unreserved_epoch(context.epoch_root) + assert process.wait(timeout=30) == 2, "the complete issuer did not refuse the injected original condition" + assert "One consumed recovery epoch grant send completed" not in log_path.read_text() + if started: + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "refused target exits without native state", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + consumer_log = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" + assert "Recovery custody acknowledged" not in consumer_log.read_text() + _no_target_child(target.generation, running=False) + finally: + if process is not None and process.poll() is None: + process.kill() + process.wait(timeout=30) + if started and Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "true": + Chain.docker("kill", target.generation.container) + assert_unreserved_epoch(context.epoch_root) + + +def _original_negatives(context: NativeNegativeContext, target: PreparedTarget, chain: Chain, preserved_snapshot: Callable[[], object]) -> list[str]: + directory = context.fixture_root / "native-original-negative-backups" + directory.mkdir(mode=0o700) + database = chain.generations[0].path("journal") / "management-operations.sqlite3" + saved = directory / "absent-original.sqlite3" + baseline = preserved_snapshot() + original = (database.stat().st_dev, database.stat().st_ino, _digest(database)) + database.rename(saved) + try: + injected = preserved_snapshot() + _issuer_refusal(context, target, "original-journal-absence-refused") + assert preserved_snapshot() == injected + finally: + assert_unreserved_epoch(context.epoch_root) + assert not database.exists() and (saved.stat().st_dev, saved.stat().st_ino, _digest(saved)) == original + saved.rename(database) + assert preserved_snapshot() == baseline + for label, path in (("changed-installed-ready-refused", Path(target.receipt.environment_path).parent / "MANAGEMENT_READY.json"), ("changed-target-management-config-refused", target.generation.path("bootstrap") / "management.yaml")): + baseline = preserved_snapshot() + with _replacement(context.epoch_root, path, directory / label, path.read_bytes() + b"\n"): + injected = preserved_snapshot() + _issuer_refusal(context, target, label) + assert preserved_snapshot() == injected + assert preserved_snapshot() == baseline + return sorted(ORIGINAL_NEGATIVE_CASES) + + +def _reserved_contender() -> None: + assert set(os.listdir(EPOCH_ROOT)) == {"inventory.json", "epoch.lock", "reserved.json", "reserved-head.json"} + before = _snapshot((EPOCH_ROOT,)) + contender = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert contender.returncode == 2 and _snapshot((EPOCH_ROOT,)) == before, "a contender overlapped the actual original durable reservation" + + +def _retarget_reserved(barrier: KernelPublicationBarrier, issuer: subprocess.Popen, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: + inventory = EPOCH_ROOT / "inventory.json" + original_digest = _digest(inventory) + raw = json.loads(inventory.read_bytes()) + raw["preparation_id"] = "foreign-" + uuid.uuid4().hex + raw["target_id"] = ("f" if raw["target_id"] != "f" * 64 else "e") * 64 + with inventory.open("wb") as stream: + stream.write(canonical(raw)) + stream.flush() + os.fsync(stream.fileno()) + _reserved_contender() + injected = _snapshot((EPOCH_ROOT,)) + barrier.release() + issuer.wait(timeout=120) + assert issuer.returncode == 2 and _snapshot((EPOCH_ROOT,)) == injected + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "retargeted reservation ends current custody") + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() and "One consumed recovery epoch grant send completed" not in issuer_log.read_text() + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == injected + return {"phase": "reserved", "consumer_grant_acknowledged": False, "inventory_sha256": original_digest, "retargeted_inventory_sha256": _digest(inventory), "retained_publications": ["reserved.json", "reserved-head.json"]} + + def run_fixture(*, authorization_id: str, source_repository: Path, source_token_file: Path, source_sha: str, uv: Path, uv_version: str, output: Path, scenario: str = "grant") -> dict[str, object]: required_cases(scenario) if not re.fullmatch(r"[0-9a-f]{40}", source_sha) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", uv_version): @@ -549,6 +694,8 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ for directory in (FIXTURE_ROOT, EPOCH_ROOT, PEER_ROOT, AUTHORIZATION_ROOT): if directory.exists() or directory.is_symlink(): raise FileExistsError("recovery fixture refuses a previous/partial root: " + str(directory)) + for parent in (Path("/var/lib/auto-coder"), Path("/etc/auto-coder")): + _private_directory(parent) _private_directory(FIXTURE_ROOT) root = FIXTURE_ROOT / uuid.uuid4().hex root.mkdir(mode=0o700) @@ -567,6 +714,7 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ issuer = None log_stream = None target = None + barrier = None try: stock = InhibitedStockRoutes.install() worker_root = root / "unrelated" @@ -618,13 +766,26 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ routes.assert_inhibited() stock.assert_inhibited() selected_target = target_from_container(target.generation.container, bootstrap_config=target.generation.path("bootstrap") / "config.json", management_config=target.generation.path("bootstrap") / "management.yaml", published_port=target.published_port) - enroll_recovery_scope(tuple(captures), selected_target, tuple(watchers), deployment_id=chain.history_binding.deployment, enrollment_id=target.enrollment_id, authorization=authorization) + enrollment = enroll_recovery_scope(tuple(captures), selected_target, tuple(watchers), deployment_id=chain.history_binding.deployment, enrollment_id=target.enrollment_id, authorization=authorization) _install_authorization(chain, target, authorization_id, source_repository, source_token_file, source_sha, uv, uv_version) retained_roots = (Path(chain.history_binding.root), chain.oauth, *(path for generation in chain.generations for path in (generation.path("state"), generation.path("journal"), generation.path("bootstrap")))) Chain.checkpoint(chain.oauth / "oauth.sqlite3") preserved = _snapshot(retained_roots) worker_before = (worker_baseline, worker_posts, worker_claims) workers.preserved(*worker_before) + native_context = NativeNegativeContext(enrollment.enrollment, root, EPOCH_ROOT) + if scenario == "grant": + + def preservation() -> object: + workers.preserved(*worker_before) + return _snapshot(retained_roots, semantic=True), worker_claims, workers.effect_posts() + + cases.extend(LINEAGE_REPORT_CASES) + cases.extend(run_native_negatives(native_context, lambda label: _issuer_refusal(native_context, target, label), preservation)) + cases.extend(_original_negatives(native_context, target, chain, preservation)) + # Only the intentional reversible fixture injections changed path + # metadata. This fresh strict baseline precedes actual issuance. + preserved = _snapshot(retained_roots) issuer_log = root / "issuer.log" log_stream = issuer_log.open("xb") issuer = subprocess.Popen([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], stdout=log_stream, stderr=subprocess.STDOUT, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) @@ -634,17 +795,47 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ cases.append("independent-issuer-contender-refused") # A real SIGSTOP barrier lets the exact target execute its installed # interpreter probe before T takes its current process observations. - issuer.send_signal(signal.SIGSTOP) - _wait(lambda: Path(f"/proc/{issuer.pid}/stat").read_text().rsplit(")", 1)[1].split()[0] == "T", "issuer kernel stop barrier", process=issuer) + if scenario in BOUNDARY_SCENARIOS or scenario in ("grant", "retarget-after-reservation"): + publication, when = parse_boundary(scenario) if scenario in BOUNDARY_SCENARIOS else ("reserved-head.json", "after") + barrier = KernelPublicationBarrier(issuer, root=EPOCH_ROOT, name=publication, when=when) + barrier.start() + else: + issuer.send_signal(signal.SIGSTOP) + _wait(lambda: Path(f"/proc/{issuer.pid}/stat").read_text().rsplit(")", 1)[1].split()[0] == "T", "issuer kernel stop barrier", process=issuer) chain.docker("start", target.generation.container) probe = json.loads(chain.docker("exec", target.generation.container, target.receipt.interpreter_path, "-I", "-B", "-m", "auto_coder.build_provenance", timeout=60).splitlines()[-1]) assert probe["source_revision"]["available"] is True and probe["source_revision"]["value"] == source_sha assert probe["distribution_version"]["available"] is True and probe["distribution_version"]["value"] == target.receipt.distribution_version + _wait(lambda: chain.docker("inspect", "--format", "{{json .ExecIDs}}", target.generation.container) in ("null", "[]"), "exact target probe has left no active Docker exec") _no_target_child(target.generation, running=True) cases.append("genuine-installed-ready-in-target") - issuer.send_signal(signal.SIGCONT) + if scenario == "grant": + cases.extend(verify_native_containment_reports(native_context, issuer.pid)) + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) consumer_log = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" result["source_proof"] = {name: getattr(target.receipt, name) for name in ("source_sha", "tested_source_ref", "tested_source_object", "test_run_id", "test_run_attempt", "lock_sha256", "distribution_version", "receipt_identity")} + if barrier is not None: + barrier.resume_initial() + observed_boundary = barrier.wait(timeout=1800) + if scenario == "grant": + _reserved_contender() + cases.append("reservation-barrier-contender-refused") + result["reservation_barrier"] = asdict(observed_boundary) + barrier.release() + else: + if scenario == "retarget-after-reservation": + result["durable"] = _retarget_reserved(barrier, issuer, issuer_log, consumer_log, target) + else: + barrier.kill() + result["durable"] = _publication_interruption(scenario, observed_boundary, issuer_log, consumer_log, target) + cases.extend(("native-" + scenario, "interrupted-epoch-replay-refused")) + assert _snapshot(retained_roots) == preserved + workers.preserved(*worker_before) + _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) + return result + else: + issuer.send_signal(signal.SIGCONT) if scenario != "grant": result["durable"] = _interrupted_issuer(scenario, issuer, issuer_log, consumer_log, target) cases.extend(("native-" + scenario, "interrupted-epoch-replay-refused")) @@ -652,7 +843,7 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ workers.preserved(*worker_before) _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) return result - _wait(lambda: (EPOCH_ROOT / "receipt-head.json").is_file() and consumer_log.is_file() and "Recovery custody acknowledged" in consumer_log.read_text(), "native grant and consumer custody", timeout=600, process=issuer) + _wait(lambda: (EPOCH_ROOT / "receipt-head.json").is_file() and consumer_log.is_file() and "Recovery custody acknowledged" in consumer_log.read_text(), "native grant and consumer custody", timeout=1800, process=issuer) result["durable"] = inspect_durable_result(EPOCH_ROOT, issuer_log=issuer_log.read_text(), consumer_log=consumer_log.read_text()) cases.append("complete-native-epoch-granted") for captured in captures: @@ -706,6 +897,8 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ result["diagnostics"] = diagnostics[-120:] output.parent.mkdir(parents=True, exist_ok=True) _write_private(output, canonical(result)) + if barrier is not None: + barrier.close() if issuer is not None and issuer.poll() is None: issuer.send_signal(signal.SIGCONT) issuer.terminate() diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index b34318119..9dc037fb2 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -179,7 +179,7 @@ def _result() -> dict[str, object]: return { "schema": fixture.RESULT_SCHEMA, "status": "passed", - "cases": sorted(fixture.REQUIRED_CASES), + "cases": sorted(fixture.required_cases("grant")), "no_child_started": True, "preserved_originals": True, "preserved_workers": True, @@ -188,7 +188,7 @@ def _result() -> dict[str, object]: } -@pytest.mark.parametrize("case", sorted(fixture.REQUIRED_CASES)) +@pytest.mark.parametrize("case", sorted(fixture.required_cases("grant"))) def test_native_result_cannot_omit_any_required_case(case: str) -> None: result = _result() fixture.validate_result(result) @@ -221,3 +221,38 @@ def test_interruption_result_requires_the_actual_boundary_and_no_grant(scenario: def test_unknown_native_scenario_is_not_a_success_alias() -> None: with pytest.raises(ValueError, match="unsupported"): fixture.required_cases("unreviewed-boundary") + + +@pytest.mark.parametrize("scenario", fixture.BOUNDARY_SCENARIOS) +def test_publication_result_requires_exact_kernel_observation(scenario: str) -> None: + publication, when = fixture.parse_boundary(scenario) + result = _result() + result.update(scenario=scenario, cases=sorted(fixture.required_cases(scenario)), durable={"publication": publication, "when": when, "consumer_grant_acknowledged": False, "file_fsync_observed": when == "after", "directory_fsync_observed": when == "after"}) + fixture.validate_result(result) + result["durable"]["directory_fsync_observed"] = when != "after" + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +def test_retarget_result_cannot_hide_unchanged_inventory() -> None: + result = _result() + result.update(scenario="retarget-after-reservation", cases=sorted(fixture.required_cases("retarget-after-reservation")), durable={"phase": "reserved", "consumer_grant_acknowledged": False, "inventory_sha256": "a" * 64, "retargeted_inventory_sha256": "a" * 64}) + with pytest.raises(AssertionError): + fixture.validate_result(result) + result["durable"]["retargeted_inventory_sha256"] = "b" * 64 + fixture.validate_result(result) + + +def test_semantic_preservation_excludes_only_deliberate_rename_metadata(tmp_path: Path) -> None: + original = tmp_path / "original" + original.mkdir() + database = original / "database" + database.write_bytes(b"retained bytes") + baseline = fixture._snapshot((original,), semantic=True) + held = tmp_path / "held-original" + database.rename(held) + assert fixture._snapshot((original,), semantic=True) != baseline + held.rename(database) + assert fixture._snapshot((original,), semantic=True) == baseline + database.write_bytes(b"changed bytes") + assert fixture._snapshot((original,), semantic=True) != baseline diff --git a/tests/test_management_recovery_epoch_native_negatives.py b/tests/test_management_recovery_epoch_native_negatives.py new file mode 100644 index 000000000..32c9cb155 --- /dev/null +++ b/tests/test_management_recovery_epoch_native_negatives.py @@ -0,0 +1,278 @@ +"""Real-file/process safety guards for the native negative fixture helper. + +These component checks do not claim Docker/systemd or complete issuer acceptance. +The separately authorized VM runs every production-issuer and report-CLI case. +""" + +from __future__ import annotations + +import fcntl +import os +import signal +import stat +import subprocess +import sys +from contextlib import nullcontext +from dataclasses import replace +from pathlib import Path + +import pytest +from scripts import recovery_epoch_native_negatives as fixture +from scripts.test_management_containment_enrollment_host import _write_private + +from auto_coder.management.containment_enrollment import OriginalNetworkPin +from auto_coder.management.containment_kernel_subobservation import _holder_start_token +from auto_coder.management.recovery_containment_model import RecoveryLock +from tests.test_management_recovery_containment import _enrollment + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@pytest.fixture +def _use_real_commands(): + yield + + +@pytest.fixture +def _use_real_sleep(): + yield + + +def _context(tmp_path: Path) -> fixture.NativeNegativeContext: + epoch = tmp_path / "epoch" + epoch.mkdir(mode=0o700) + _write_private(epoch / "epoch.lock", b"") + _write_private(epoch / "inventory.json", b'{"component":"no-authority"}') + return fixture.NativeNegativeContext(_enrollment(), tmp_path, epoch) + + +def test_unreserved_guard_observes_but_never_rewrites_original_inputs(tmp_path: Path) -> None: + context = _context(tmp_path) + before = {path.name: fixture._projection(path) for path in context.epoch_root.iterdir()} + assert fixture._epoch_snapshot(context.epoch_root) == fixture._epoch_snapshot(context.epoch_root) + fixture.assert_unreserved_epoch(context.epoch_root) + assert {path.name: fixture._projection(path) for path in context.epoch_root.iterdir()} == before + + +@pytest.mark.parametrize("name", ["reserved.json", "reserved-head.json", "intent.json", "intent-head.json", "consumed.json", "consumed-head.json", "receipt.json", "receipt-head.json", ".partial", "unexpected.json"]) +def test_any_original_publication_or_partial_name_permanently_aborts_fixture(tmp_path: Path, name: str) -> None: + context = _context(tmp_path) + path = context.epoch_root / name + _write_private(path, b"uncertain original bytes") + before = {item.name: fixture._projection(item) for item in context.epoch_root.iterdir()} + with pytest.raises(fixture.EpochAlreadyPublished, match="never repair or retry"): + fixture.assert_unreserved_epoch(context.epoch_root) + assert {item.name: fixture._projection(item) for item in context.epoch_root.iterdir()} == before + + +@pytest.mark.parametrize("change", ["symlink", "hardlink", "file-mode", "directory-mode", "missing-input"]) +def test_unreserved_guard_requires_bounded_original_objects(tmp_path: Path, change: str) -> None: + context = _context(tmp_path) + lock = context.epoch_root / "epoch.lock" + if change == "symlink": + lock.unlink() + lock.symlink_to(context.epoch_root / "inventory.json") + elif change == "hardlink": + os.link(lock, tmp_path / "alias") + elif change == "file-mode": + lock.chmod(0o640) + elif change == "directory-mode": + context.epoch_root.chmod(0o750) + else: + lock.unlink() + with pytest.raises(fixture.EpochAlreadyPublished): + fixture.assert_unreserved_epoch(context.epoch_root) + + +def test_epoch_inode_injection_restores_the_same_original_only_before_publication(tmp_path: Path) -> None: + context = _context(tmp_path) + original = context.epoch_root / "epoch.lock" + before = fixture._projection(original) + backup = tmp_path / "epoch.original" + inventory = fixture._projection(context.epoch_root / "inventory.json") + with fixture._replacement(context.epoch_root, original, backup, b""): + assert fixture._projection(original)[:2] != before[:2] + assert fixture._projection(backup) == before + fixture.assert_unreserved_epoch(context.epoch_root) + assert fixture._projection(original) == before + assert fixture._projection(context.epoch_root / "inventory.json") == inventory + assert not backup.exists() + + +def test_publication_during_epoch_injection_keeps_every_original_and_never_resets(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = context.epoch_root / "epoch.lock", tmp_path / "epoch.original" + original = fixture._projection(path) + with pytest.raises(fixture.EpochAlreadyPublished, match="never repair or retry"): + with fixture._replacement(context.epoch_root, path, backup, b""): + injected = fixture._projection(path) + _write_private(context.epoch_root / "reserved.json", b"original partial record") + assert fixture._projection(path) == injected + assert fixture._projection(backup) == original + assert (context.epoch_root / "reserved.json").read_bytes() == b"original partial record" + assert {item.name for item in context.epoch_root.iterdir()} == {"epoch.lock", "inventory.json", "reserved.json"} + + +def test_modified_injection_is_retained_instead_of_destructively_restored(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = tmp_path / "owner", tmp_path / "owner.original" + _write_private(path, b"original owner") + original = fixture._projection(path) + with pytest.raises(AssertionError, match="replacement changed"): + with fixture._replacement(context.epoch_root, path, backup, b"injected owner"): + path.write_bytes(b"unexpected issuer mutation") + assert path.read_bytes() == b"unexpected issuer mutation" + assert fixture._projection(backup) == original + + +def test_each_refusal_compares_injected_and_restored_original_state(tmp_path: Path) -> None: + context = _context(tmp_path) + original, backup = tmp_path / "owner", tmp_path / "owner.original" + _write_private(original, b"native original") + before = fixture._projection(original) + calls = [] + + def refusal(label: str) -> None: + assert fixture._projection(original)[:2] != before[:2] + calls.append(label) + + fixture._run_case(context, "component-refusal", lambda: fixture._replacement(context.epoch_root, original, backup, b"native original"), refusal, lambda: fixture._projection(original)) + assert calls == ["component-refusal"] + assert fixture._projection(original) == before + + +def test_refused_issuer_cannot_modify_unrelated_original_state(tmp_path: Path) -> None: + context = _context(tmp_path) + path = tmp_path / "original-operation-store" + _write_private(path, b"original") + + def invalid_refusal(_: str) -> None: + path.write_bytes(b"unexpected change") + + with pytest.raises(AssertionError, match="changed original/OAuth/worker"): + fixture._run_case(context, "component-refusal", nullcontext, invalid_refusal, lambda: fixture._projection(path)) + assert path.read_bytes() == b"unexpected change" + + +def test_refused_issuer_cannot_modify_epoch_inventory_without_a_record(tmp_path: Path) -> None: + context = _context(tmp_path) + + def invalid_refusal(_: str) -> None: + (context.epoch_root / "inventory.json").write_bytes(b"unexpected replacement configuration") + + with pytest.raises(AssertionError, match="changed an original epoch input"): + fixture._run_case(context, "component-refusal", nullcontext, invalid_refusal, lambda: "unchanged original state") + assert (context.epoch_root / "inventory.json").read_bytes() == b"unexpected replacement configuration" + + +def test_actual_ex_contender_prevents_independent_shared_fence_and_releases_only_itself(tmp_path: Path) -> None: + path = tmp_path / "original.lock" + _write_private(path, b"") + info = path.stat() + selected = RecoveryLock(str(path), info.st_dev, info.st_ino) + code = "import fcntl,sys;f=open(sys.argv[1],'rb');\ntry: fcntl.flock(f,fcntl.LOCK_SH|fcntl.LOCK_NB)\nexcept BlockingIOError: sys.exit(7)\n" + with fixture._busy_fence(selected): + assert subprocess.run((sys.executable, "-I", "-B", "-c", code, str(path)), timeout=10).returncode == 7 + assert subprocess.run((sys.executable, "-I", "-B", "-c", code, str(path)), timeout=10).returncode == 0 + assert path.stat().st_ino == selected.inode and path.read_bytes() == b"" + + +def test_actual_busy_fence_does_not_accept_a_different_selected_inode(tmp_path: Path) -> None: + path = tmp_path / "owner.lock" + _write_private(path, b"") + info = path.stat() + with pytest.raises(AssertionError, match="selected original"): + with fixture._busy_fence(RecoveryLock(str(path), info.st_dev, info.st_ino + 1)): + pytest.fail("a different original identity was accepted") + with path.open("rb") as stream: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + + +def test_real_process_barrier_and_pidfd_cleanup_leave_no_running_fixture_child() -> None: + code = "import sys;print('READY',flush=True);sys.stdin.buffer.read(1)" + with fixture._live_process((sys.executable, "-I", "-B", "-c", code)) as process: + pid = process.pid + assert process.poll() is None and Path(f"/proc/{pid}").exists() + assert process.returncode is not None + assert not Path(f"/proc/{pid}").exists() + + +def test_malformed_real_process_readiness_is_a_failure() -> None: + with pytest.raises(AssertionError, match="exact readiness"): + with fixture._live_process((sys.executable, "-I", "-B", "-c", "print('UNAVAILABLE',flush=True)")): + pytest.fail("an invalid native barrier was accepted") + + +_RESOURCE_HOLDER = """ +import fcntl +import os +import sys +owner = open(sys.argv[1], 'rb') +fcntl.flock(owner, fcntl.LOCK_SH | fcntl.LOCK_NB) +controller = open(sys.argv[2], 'rb') +fcntl.flock(controller, fcntl.LOCK_EX | fcntl.LOCK_NB) +namespace = os.open('/proc/self/ns/net', os.O_RDONLY) +extra = open(sys.argv[1], 'rb') if sys.argv[3] == 'duplicate' else None +print('READY', flush=True) +sys.stdin.buffer.read(1) +""" + + +@pytest.mark.parametrize("duplicate", [False, True]) +def test_report_probe_discovers_actual_stopped_process_descriptors_without_inventing_fences(tmp_path: Path, duplicate: bool) -> None: + owner, controller = tmp_path / "owner.lock", tmp_path / "controller.lock" + for path in (owner, controller): + _write_private(path, b"") + namespace = os.open("/proc/self/ns/net", os.O_RDONLY | os.O_CLOEXEC) + try: + ns = os.fstat(namespace) + pin = OriginalNetworkPin(os.getpid(), _holder_start_token(os.getpid()), Path("/proc/sys/kernel/random/boot_id").read_text().strip(), namespace, os.readlink(f"/proc/self/fd/{namespace}"), ns.st_dev, ns.st_ino) + selected = _enrollment() + original = replace(selected.originals[0], owner=RecoveryLock(str(owner), owner.stat().st_dev, owner.stat().st_ino), namespace_pin=pin) + watcher = replace(selected.watchers[0], lock=RecoveryLock(str(controller), controller.stat().st_dev, controller.stat().st_ino)) + selected = replace(selected, originals=(original,), watchers=(watcher,)) + with fixture._live_process((sys.executable, "-I", "-B", "-c", _RESOURCE_HOLDER, str(owner), str(controller), "duplicate" if duplicate else "one")) as process: + with pytest.raises(AssertionError, match="SIGSTOP"): + fixture._issuer_resources(selected, process.pid) + process.send_signal(signal.SIGSTOP) + waited, status = os.waitpid(process.pid, os.WUNTRACED) + assert waited == process.pid and os.WIFSTOPPED(status) + try: + if duplicate: + with pytest.raises(AssertionError, match="absent or ambiguous"): + fixture._issuer_resources(selected, process.pid) + else: + actual = fixture._issuer_resources(selected, process.pid) + assert actual.holder_pid == process.pid and actual.holder_start_token == _holder_start_token(process.pid) + for values, expected in ((actual.original_owner_fds, owner), (actual.controller_fds, controller)): + assert len(values) == 1 + entry = Path(f"/proc/{process.pid}/fd/{values[0].descriptor}") + assert (entry.stat().st_dev, entry.stat().st_ino) == (expected.stat().st_dev, expected.stat().st_ino) + assert len(actual.namespace_fds) == 1 + assert os.readlink(f"/proc/{process.pid}/fd/{actual.namespace_fds[0].descriptor}") == pin.namespace + finally: + process.send_signal(signal.SIGCONT) + finally: + os.close(namespace) + + +@pytest.mark.parametrize("action", ["negatives", "reports"]) +def test_native_entry_refuses_unapproved_roots_before_any_fixture_effect(tmp_path: Path, action: str) -> None: + context = _context(tmp_path) + before = fixture._epoch_snapshot(context.epoch_root) + with pytest.raises(PermissionError, match="fixed disposable fixture roots"): + if action == "negatives": + fixture.run_native_negatives(context, lambda _: pytest.fail("an unauthorized issuer was started"), lambda: pytest.fail("unauthorized fixture state was sampled")) + else: + fixture.verify_native_containment_reports(context, os.getpid()) + assert fixture._epoch_snapshot(context.epoch_root) == before + assert {path.name for path in tmp_path.iterdir()} == {"epoch"} + + +def test_native_case_oracles_distinguish_actual_issuer_and_separate_report_work() -> None: + assert len(fixture.REQUIRED_NEGATIVE_CASES) == 9 + assert len(fixture.REQUIRED_REPORT_CASES) == 4 + assert fixture.REQUIRED_NEGATIVE_CASES.isdisjoint(fixture.REQUIRED_REPORT_CASES) + assert {"busy-original-owner-refused", "busy-controller-fence-refused", "replaced-epoch-lock-refused", "retained-old-network-task-and-listener-refused", "old-published-listener-refused"} <= fixture.REQUIRED_NEGATIVE_CASES + assert {"forged-native-containment-refused", "cross-deployment-containment-refused", "stale-native-containment-refused"} <= fixture.REQUIRED_REPORT_CASES + assert stat.S_IMODE(Path(fixture.__file__).stat().st_mode) & 0o111 == 0 From 88496527438b13a7d74ba866b4cc9ac43e38b2a0 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 01:25:45 +0900 Subject: [PATCH 05/24] test(management): require complete native recovery acceptance scenarios --- .../workflows/management-recovery-epoch.yml | 52 +- .../management-original-genesis.md | 23 + scripts/recovery_epoch_native_peer.py | 722 ++++++++++++++++++ scripts/recovery_vm/README.md | 29 +- scripts/recovery_vm/guest.sh | 18 + scripts/recovery_vm/prepare.sh | 26 +- .../test_management_recovery_epoch_host.py | 202 +++-- ...t_management_archived_cutover_container.py | 2 +- tests/test_management_recovery_epoch_host.py | 71 +- ...t_management_recovery_epoch_native_peer.py | 172 +++++ 10 files changed, 1250 insertions(+), 67 deletions(-) create mode 100644 scripts/recovery_epoch_native_peer.py create mode 100644 tests/test_management_recovery_epoch_native_peer.py diff --git a/.github/workflows/management-recovery-epoch.yml b/.github/workflows/management-recovery-epoch.yml index 636972336..d243b4287 100644 --- a/.github/workflows/management-recovery-epoch.yml +++ b/.github/workflows/management-recovery-epoch.yml @@ -12,14 +12,50 @@ concurrency: cancel-in-progress: true jobs: + acceptance-plan: + name: Select the complete native recovery acceptance matrix + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + scenarios: ${{ steps.scenarios.outputs.scenarios }} + steps: + - name: Checkout exact reviewed source + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.13.6" + - name: Install locked test dependencies + run: uv sync --locked --dev --extra test + - name: Select every independent production scenario + id: scenarios + run: | + .venv/bin/python -B - <<'PY' + import json + import os + from scripts.test_management_recovery_epoch_host import ACCEPTANCE_SCENARIOS, required_cases + + assert len(ACCEPTANCE_SCENARIOS) == len(set(ACCEPTANCE_SCENARIOS)) + assert all(required_cases(scenario) for scenario in ACCEPTANCE_SCENARIOS) + with open(os.environ["GITHUB_OUTPUT"], "a") as stream: + stream.write("scenarios=" + json.dumps(ACCEPTANCE_SCENARIOS) + "\n") + PY recovery-epoch: name: Recovery epoch (${{ matrix.scenario }}) in isolated VM + needs: acceptance-plan runs-on: ubuntu-24.04 timeout-minutes: 150 strategy: fail-fast: false + max-parallel: 4 matrix: - scenario: [grant, kill-after-intent, kill-after-consume, lost-reply] + scenario: ${{ fromJSON(needs.acceptance-plan.outputs.scenarios) }} env: UV_LOCKED: "1" PYTHONDONTWRITEBYTECODE: "1" @@ -85,3 +121,17 @@ jobs: path: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }}/output/ if-no-files-found: error retention-days: 7 + acceptance: + name: Management Recovery Epoch Acceptance + needs: [acceptance-plan, recovery-epoch] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 2 + steps: + - name: Require every native scenario to succeed + env: + PLAN_RESULT: ${{ needs.acceptance-plan.result }} + ACCEPTANCE_RESULT: ${{ needs.recovery-epoch.result }} + run: | + test "$PLAN_RESULT" = success + test "$ACCEPTANCE_RESULT" = success diff --git a/docs/client-features/management-original-genesis.md b/docs/client-features/management-original-genesis.md index fd2b30fbf..729496b8c 100644 --- a/docs/client-features/management-original-genesis.md +++ b/docs/client-features/management-original-genesis.md @@ -61,6 +61,22 @@ its exact protected configuration bind mount, and the manager's network namespace. All original files are owner-only regular files with one link; protected original directories have mode `0700`. +Capture parses the effective management settings, rather than accepting only a +configuration hash. Its OAuth root, operation journal root, bootstrap path and +listener must select those same original objects. The mounted bootstrap repeats +that check before diagnostics can create a missing original directory and before +any native store or launch record. It also verifies the selected initial +executable receipt, the installed management console wrapper and its supported +environment. A missing original root or an executable environment hook refuses +without reconstructing the missing state. + +The ordinary proxy may use its explicit owner-only writable `/data` and `/config` +bind roots as a pair. The ordinary predecessor capture checks their exact +canonical host directories; anonymous volumes or a lone additional data mount +do not meet this profile. A genesis predecessor carries the same read-only +`/management-original-history` mount into the existing successor capture, whose +host reader verifies that binding against the original root authority. + Root authorization JSON contains exactly the fields of `GenesisRootAuthorization`: `profile` (`ordinary-original-roots-v1`), `purpose` (`separately-authorized-new-deployment`), `deployment_id`, canonical UUID @@ -169,6 +185,13 @@ outcomes, complete failed branches, and missing-original refusal. Docker observations and container bind mapping are explicit component substitutes; these tests are not real-Docker, first-child or recovery-epoch acceptance. +`tests/test_management_original_genesis_cli.py` invokes the actual bootstrap CLI +against a missing original state root and proves that logger setup does not +recreate it. The component selection also checks effective configuration mismatches +and executable/environment failures before native publication. The additive +predecessor and explicit proxy-data capture checks are exercised in +`tests/test_management_original_genesis_capture.py`. + The production interfaces are suitable for isolated real-Docker and host-root acceptance. Full #2824 acceptance additionally requires the actual complete current containment verifier, issuer, OS descriptor transfer, consumer receipt diff --git a/scripts/recovery_epoch_native_peer.py b/scripts/recovery_epoch_native_peer.py new file mode 100644 index 000000000..c874f38a9 --- /dev/null +++ b/scripts/recovery_epoch_native_peer.py @@ -0,0 +1,722 @@ +"""Real peer and late-mutation faults in the separately authorized recovery VM. + +Only the original production issuer and bootstrap can produce authority. This +fixture observes real syscalls, retained process lifetimes and received kernel +objects. It may stop/kill its exact target or mutate an explicitly selected +fixture object; it never changes syscall arguments/results or production code. +""" + +from __future__ import annotations + +import array +import ctypes +import fcntl +import hashlib +import json +import os +import select +import signal +import socket +import stat +import subprocess +import sys +import time +from contextlib import ExitStack, contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable, Iterator + +from scripts.recovery_epoch_fault_tracer import _AUDIT_ARCH_X86_64, _PTRACE_GET_SYSCALL_INFO, KernelPublicationBarrier, TraceProcess, _Information, _token +from scripts.recovery_epoch_native_negatives import NativeNegativeContext, _epoch_snapshot, _projection, assert_unreserved_epoch, require_native_context +from scripts.test_management_containment_enrollment_host import _docker, _systemctl + +from auto_coder.logger_config import get_logger +from auto_coder.management.bootstrap_supervisor import BootstrapConfig +from auto_coder.management.containment_route_manifest import _systemd_unit +from auto_coder.management.original_attempts import canonical +from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, PEER_SOCKET, EpochAuthorization +from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA, ConsumerConfiguration, parse_consumer_configuration +from auto_coder.management.recovery_epoch_peer import CHALLENGE_SCHEMA, MAX_MESSAGE_BYTES, MAX_RESOURCES, OFFER_SCHEMA, RECEIPT_SCHEMA, inspect_kernel_lock + +logger = get_logger(__name__) +REQUIRED_PEER_CASES = frozenset({"queued-rights-without-receipt-refused", "received-rights-without-receipt-refused", "changed-target-lifetime-refused", "foreign-acquirer-sh-refused"}) +LATE_MUTATION_SCENARIOS = ( + "late-management-config", + "late-ready-receipt", + "late-installed-source", + "late-original-journal", + "late-original-attempt", + "late-watcher-config", + "late-controller-restarter", + "late-original-fence", + "late-target-fence", + "late-controller-fence", +) +_SYS_SENDTO = 44 +_SYS_SENDMSG = 46 +_SYS_RECVMSG = 47 +_SYS_PIDFD_GETFD = 438 + + +def _wait(predicate: Callable[[], bool], description: str, *, timeout: float = 30) -> None: + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() >= deadline: + raise AssertionError("native peer barrier timed out: " + description) + time.sleep(0.02) + + +def _target_state(context: NativeNegativeContext) -> dict[str, object]: + value = json.loads(_docker("inspect", "--format", "{{json .State}}", context.enrollment.target.management_id)) + if not isinstance(value, dict): + raise AssertionError("actual target Docker state is unavailable") + return value + + +class DockerTraceProcess: + """Pin one actual Docker PID without inventing a parent/child relationship.""" + + def __init__(self, context: NativeNegativeContext): + self.context = context + state = _target_state(context) + if state.get("Running") is not True or type(state.get("Pid")) is not int or state["Pid"] <= 1: + raise AssertionError("the exact production target has no current lifetime") + self.pid = int(state["Pid"]) + self.start_token = _token(self.pid) + self.docker_started_at = state.get("StartedAt") + self.returncode: int | None = None + self.pidfd = os.pidfd_open(self.pid, 0) + self.paused = False + self.check() + + def dead(self) -> bool: + poll = select.poll() + poll.register(self.pidfd, select.POLLIN) + return bool(poll.poll(0)) + + def check(self) -> None: + if self.pidfd < 0 or self.dead() or _token(self.pid) != self.start_token: + raise AssertionError("the exact selected target lifetime ended or changed") + state = _target_state(self.context) + if state.get("Running") is not True or state.get("Pid") != self.pid or state.get("StartedAt") != self.docker_started_at: + raise AssertionError("the selected Docker object now names another lifetime") + + def wait(self, timeout: float | None = None) -> int: + """Observe Docker's actual termination after tracer wait ownership ends.""" + timeout = 30 if timeout is None else timeout + poll = select.poll() + poll.register(self.pidfd, select.POLLIN) + if not poll.poll(max(1, int(timeout * 1000))): + raise subprocess.TimeoutExpired("actual target pidfd", timeout) + if self.returncode is None: + _wait(lambda: _target_state(self.context).get("Running") is False, "Docker records the original target's exit", timeout=timeout) + state = _target_state(self.context) + if state.get("StartedAt") != self.docker_started_at or type(state.get("ExitCode")) is not int: + raise AssertionError("Docker no longer retains the original target exit code") + self.returncode = int(state["ExitCode"]) + return self.returncode + + def pause(self) -> None: + self.check() + signal.pidfd_send_signal(self.pidfd, signal.SIGSTOP) + _wait(lambda: Path(f"/proc/{self.pid}/stat").read_text().rsplit(")", 1)[1].split()[0] in {"T", "t"}, "exact target is kernel-stopped") + self.check() + self.paused = True + + def resume(self) -> None: + if self.paused and not self.dead(): + self.check() + signal.pidfd_send_signal(self.pidfd, signal.SIGCONT) + self.paused = False + + def kill(self) -> None: + self.check() + signal.pidfd_send_signal(self.pidfd, signal.SIGKILL) + + def close(self) -> None: + if self.pidfd >= 0: + self.resume() + os.close(self.pidfd) + self.pidfd = -1 + + +def _consumer_configuration(context: NativeNegativeContext) -> ConsumerConfiguration: + selected = [item for item in context.enrollment.target.mounts if item.destination == "/bootstrap/recovery-epoch.json"] + if len(selected) != 1 or selected[0].writable: + raise AssertionError("the actual consumer lacks its fixed read-only configuration") + result = parse_consumer_configuration(Path(selected[0].source).read_bytes()) + if (result.authorization_id, result.target_id) != (context.enrollment.authorization_id, context.enrollment.target.management_id): + raise AssertionError("the selected native consumer belongs to another fixture") + return result + + +def _fd_for_object(pid: int, device: int, inode: int, mode: str) -> int: + matches = [] + for path in Path(f"/proc/{pid}/fd").iterdir(): + if not path.name.isdecimal(): + continue + try: + info = path.stat() + if (info.st_dev, info.st_ino) == (device, inode): + lock = inspect_kernel_lock(pid, int(path.name), device, inode) + if lock is not None and lock.mode == mode: + matches.append(int(path.name)) + except FileNotFoundError: + continue + if len(matches) != 1: + raise AssertionError("the actual selected process does not hold one exact fence descriptor") + return matches[0] + + +def _duplicate(pid: int, descriptor: int, device: int, inode: int) -> int: + """Duplicate the real OFD; unsupported pidfd_getfd is a fixture failure.""" + lifetime = _token(pid) + pidfd = os.pidfd_open(pid, 0) + duplicate = -1 + try: + libc = ctypes.CDLL(None, use_errno=True) + libc.syscall.restype = ctypes.c_long + duplicate = int(libc.syscall(ctypes.c_long(_SYS_PIDFD_GETFD), ctypes.c_int(pidfd), ctypes.c_int(descriptor), ctypes.c_uint(0))) + if duplicate < 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + info = os.fstat(duplicate) + if (info.st_dev, info.st_ino) != (device, inode) or _token(pid) != lifetime: + raise AssertionError("the duplicated native descriptor or process changed") + os.set_inheritable(duplicate, False) + return duplicate + except BaseException: + if duplicate >= 0: + os.close(duplicate) + raise + finally: + os.close(pidfd) + + +@contextmanager +def _foreign_acquirer(pid: int, device: int, inode: int, mode: str) -> Iterator[dict[str, object]]: + descriptor = _fd_for_object(pid, device, inode, mode) + before = inspect_kernel_lock(pid, descriptor, device, inode) + if before is None or before.pid != pid: + raise AssertionError("the selected fence was not independently held before the real fault") + duplicate = _duplicate(pid, descriptor, device, inode) + try: + fcntl.flock(duplicate, fcntl.LOCK_UN) + fcntl.flock(duplicate, (fcntl.LOCK_SH if mode == "READ" else fcntl.LOCK_EX) | fcntl.LOCK_NB) + after = inspect_kernel_lock(pid, descriptor, device, inode) + if after is None or after.pid != os.getpid() or after.mode != mode or after == before: + raise AssertionError("the real shared OFD did not record the foreign acquiring process") + yield {"pid": pid, "start_token": _token(pid), "descriptor": descriptor, "device": device, "inode": inode, "before": asdict(before), "after": asdict(after)} + if inspect_kernel_lock(os.getpid(), duplicate, device, inode) != after: + raise AssertionError("the injected native OFD custody changed during refusal") + finally: + os.close(duplicate) + + +def _require_publication_prefix(context: NativeNegativeContext, issuer_pid: int, through: str) -> None: + require_native_context(context, unreserved=False) + phases = ("reserved", "intent", "consumed", "receipt") + if through not in phases: + raise ValueError("an exact existing publication prefix is required") + chosen = phases[: phases.index(through) + 1] + expected = {"inventory.json", "epoch.lock", *(name + suffix for name in chosen for suffix in (".json", "-head.json"))} + if {path.name for path in context.epoch_root.iterdir()} != expected: + raise AssertionError("the actual issuer is not stopped at the selected original publication prefix") + inventory_sha = hashlib.sha256((context.epoch_root / "inventory.json").read_bytes()).hexdigest() + previous, binding = "", None + for sequence, phase in enumerate(chosen): + encoded = (context.epoch_root / (phase + ".json")).read_bytes() + record = json.loads(encoded) + head_bytes = (context.epoch_root / (phase + "-head.json")).read_bytes() + head = json.loads(head_bytes) + if canonical(record) != encoded or canonical(head) != head_bytes or record.get("phase") != phase or record.get("sequence") != sequence or record.get("previous") != previous or record.get("inventory_sha256") != inventory_sha: + raise AssertionError("the actual durable publication chain is malformed") + if head != {"schema_version": 1, "phase": phase, "record_sha256": hashlib.sha256(encoded).hexdigest(), "previous": previous, "inventory_sha256": inventory_sha}: + raise AssertionError("the actual publication head differs from its original record") + binding = record["binding"] if binding is None else binding + if record.get("binding") != binding: + raise AssertionError("the stopped publication prefix changed authority") + previous = hashlib.sha256(head_bytes).hexdigest() + if binding is None or (binding["host_pid"], binding["host_start_token"], binding["host_boot_id"]) != (issuer_pid, _token(issuer_pid), Path("/proc/sys/kernel/random/boot_id").read_text().strip()): + raise AssertionError("the durable original prefix belongs to another issuer lifetime") + + +def _mutation_file(context: NativeNegativeContext, scenario: str) -> Path: + target = context.enrollment.target + if scenario == "late-management-config": + return Path(target.management_config.path) + if scenario == "late-watcher-config": + return Path(context.enrollment.watchers[0].configuration.path) + if scenario == "late-original-journal": + original = context.enrollment.originals[0] + config = BootstrapConfig.read(Path(original.bootstrap_config.path)) + mounts = [item for item in original.mounts if item.container_id == original.management_id and item.destination == str(config.journal_root) and item.writable] + if len(mounts) != 1: + raise AssertionError("the original native journal mapping is ambiguous") + return Path(mounts[0].source) / "management-operations.sqlite3" + authorization = EpochAuthorization.parse(json.loads(AUTHORIZATION_FILE.read_bytes())) + if authorization.target_id != target.management_id: + raise AssertionError("the actual fixed authorization selects a different target") + if scenario == "late-original-attempt": + path = Path(authorization.history.root) / "00000000.json" + original = json.loads(path.read_bytes()) + if original.get("kind") != "anchor": + raise AssertionError("the selected first event is not an actual original attempt") + return path + if scenario == "late-ready-receipt": + return Path(authorization.ready_receipt.path) + if scenario == "late-installed-source": + receipt = json.loads(Path(authorization.ready_receipt.path).read_bytes()) + return Path(receipt["environment_path"]) / "lib/python3.12/site-packages/auto_coder/__init__.py" + raise ValueError("the selected late fault is not a file mutation") + + +@contextmanager +def mutate_after_intent(context: NativeNegativeContext, scenario: str, issuer_pid: int) -> Iterator[dict[str, object]]: + """Retain the deliberate fault through refusal and replay; never repair it.""" + if scenario not in LATE_MUTATION_SCENARIOS: + raise ValueError("a reviewed late mutation scenario is required") + _require_publication_prefix(context, issuer_pid, "intent") + observation: dict[str, object] = {"scenario": scenario, "issuer_pid": issuer_pid, "issuer_start_token": _token(issuer_pid), "boundary": "after-intent-head", "preservation_baseline": "after-deliberate-fixture-mutation"} + with ExitStack() as held: + if scenario.endswith("-fence"): + pid = issuer_pid + if scenario == "late-controller-fence": + lock, mode = context.enrollment.watchers[0].lock, "WRITE" + else: + target = DockerTraceProcess(context) + held.callback(target.close) + pid = target.pid + if scenario == "late-original-fence": + lock, mode = next(item.owner for item in context.enrollment.originals if item.owner is not None), "READ" + else: + lock, mode = context.enrollment.target.owner, "WRITE" + observation["kernel_fault"] = held.enter_context(_foreign_acquirer(pid, lock.device, lock.inode, mode)) + elif scenario == "late-controller-restarter": + unit = context.enrollment.watchers[0].unit + before = _systemd_unit(unit) + if not before.inhibited: + raise AssertionError("the actual controller was not inhibited at the original boundary") + _systemctl("unmask", unit) + _systemctl("daemon-reload") + after = _systemd_unit(unit) + if after.inhibited or after.main_pid != 0: + raise AssertionError("the real controller did not become startable without starting a process") + observation["controller_fault"] = {"unit": unit, "before": asdict(before), "after": asdict(after)} + else: + path = _mutation_file(context, scenario) + if not path.is_relative_to(context.fixture_root) or path != path.resolve(strict=True): + raise PermissionError("late mutation escaped the separately authorized fixture root") + before = _projection(path) + descriptor = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + if (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) != before[:2]: + raise AssertionError("the selected original changed before deliberate mutation") + os.write(descriptor, b"\n# deliberate native fixture source mutation\n" if scenario == "late-installed-source" else b"\x00") + os.fsync(descriptor) + finally: + os.close(descriptor) + after = _projection(path) + if before == after or before[:2] != after[:2]: + raise AssertionError("the selected native original bytes did not change in place") + observation["file_fault"] = {"path": str(path), "before": before, "after": after} + logger.bind(native_peer_fault=scenario, issuer_pid=issuer_pid).info("Injected the explicit native fault after the original intent head") + yield observation + if "file_fault" in observation and _projection(path) != after: + raise AssertionError("production refusal or replay changed the deliberately injected file") + if "controller_fault" in observation and _systemd_unit(unit) != after: + raise AssertionError("production refusal or replay changed the deliberately unmasked controller") + + +@contextmanager +def pause_target_before_grant(context: NativeNegativeContext, issuer_pid: int) -> Iterator[dict[str, object]]: + """Hold the actual receiver while the original issuer sends, then dies.""" + _require_publication_prefix(context, issuer_pid, "receipt") + target = DockerTraceProcess(context) + issuer_lifetime = os.pidfd_open(issuer_pid, 0) + try: + target.pause() + observation = {"target_pid": target.pid, "target_start_token": target.start_token, "issuer_pid": issuer_pid, "issuer_start_token": _token(issuer_pid), "boundary": "after-receipt-head-before-target-grant-observation", "issuer_death_verified_before_receiver_resumed": False} + _assert_no_target_admission(context, running_pid=target.pid) + yield observation + target.check() + issuer_dead = select.poll() + issuer_dead.register(issuer_lifetime, select.POLLIN) + if not issuer_dead.poll(0): + raise AssertionError("the exact original issuer still lives before the receiver resumes") + observation["issuer_death_verified_before_receiver_resumed"] = True + _assert_no_target_admission(context, running_pid=target.pid) + finally: + target.close() + os.close(issuer_lifetime) + + +class _Iovec(ctypes.Structure): + _fields_ = [("base", ctypes.c_void_p), ("length", ctypes.c_size_t)] + + +class _MessageHeader(ctypes.Structure): + _fields_ = [("name", ctypes.c_void_p), ("name_length", ctypes.c_uint32), ("vectors", ctypes.c_void_p), ("vector_count", ctypes.c_size_t), ("control", ctypes.c_void_p), ("control_length", ctypes.c_size_t), ("flags", ctypes.c_int)] + + +class _ControlHeader(ctypes.Structure): + _fields_ = [("length", ctypes.c_size_t), ("level", ctypes.c_int), ("kind", ctypes.c_int)] + + +@dataclass(frozen=True) +class PeerMessageObservation: + process_id: int + start_token: str + boot_id: str + schema: str + direction: str + when: str + socket_descriptor: int + socket_inode: int + message_sha256: str + bytes_observed: int + syscall_result: int | None + transferred_descriptors: tuple[int, ...] + descriptor_objects: tuple[tuple[int, int, int, str], ...] + + +def _decode_rights(control: bytes) -> tuple[int, ...]: + """Decode the bounded native Linux cmsghdr layout, never claimed JSON FDs.""" + result: list[int] = [] + offset = 0 + header_size = ctypes.sizeof(_ControlHeader) + alignment = ctypes.sizeof(ctypes.c_size_t) + while offset < len(control): + if len(control) - offset < header_size: + raise AssertionError("native SCM_RIGHTS control header is truncated") + header = _ControlHeader.from_buffer_copy(control[offset : offset + header_size]) + if header.length < header_size or offset + header.length > len(control) or (header.level, header.kind) != (socket.SOL_SOCKET, socket.SCM_RIGHTS): + raise AssertionError("native message control is not exact SCM_RIGHTS") + payload = control[offset + header_size : offset + header.length] + if len(payload) % array.array("i").itemsize: + raise AssertionError("native SCM_RIGHTS descriptor bytes are truncated") + descriptors = array.array("i") + descriptors.frombytes(payload) + result.extend(descriptors) + if len(result) > MAX_RESOURCES or any(item < 0 for item in result) or len(set(result)) != len(result): + raise AssertionError("native SCM_RIGHTS descriptor set is invalid") + end = offset + int(header.length) + aligned = (end + alignment - 1) // alignment * alignment + # Linux may include or omit the final cmsghdr alignment padding. + offset = min(aligned, len(control)) if end <= len(control) < aligned else aligned + return tuple(result) + + +class KernelPeerMessageBarrier(KernelPublicationBarrier): + """Stop one real protocol syscall with no ptrace register or memory writes.""" + + def __init__( + self, + process: TraceProcess, + *, + context: NativeNegativeContext, + schema: str, + direction: str, + when: str = "after", + before_send: Callable[[], None] | None = None, + ): + if schema not in {CHALLENGE_SCHEMA, OFFER_SCHEMA, RECEIPT_SCHEMA, GRANT_SCHEMA} or direction not in {"send", "receive"} or when not in {"before", "after"} or when == "before" and direction != "send": + raise ValueError("an exact reviewed native message boundary is required") + super().__init__(process, root=context.epoch_root, name="reserved.json", when="before") + self.context = context + self.schema = schema + self.direction = direction + self.message_when = when + self.before_send = before_send + configuration = _consumer_configuration(context) + self.binding = {name: getattr(configuration, name) for name in ("request_id", "authorization_id", "target_id")} + self._message_call: tuple[int, int, int, int] | None = None + self._selected_message: tuple[bytes, tuple[int, ...], int, int | None] | None = None + self._before_called = False + + def _memory(self, address: int, length: int) -> bytes: + if not address or type(length) is not int or not 0 <= length <= MAX_MESSAGE_BYTES + 1024: + raise AssertionError("native protocol memory read is absent or unbounded") + with Path(f"/proc/{self.issuer.pid}/mem").open("rb", buffering=0) as memory: + value = os.pread(memory.fileno(), length, address) + if len(value) != length: + raise AssertionError("native protocol memory is incomplete") + return value + + def _message(self, address: int, *, received_length: int | None) -> tuple[bytes, tuple[int, ...]]: + header = _MessageHeader.from_buffer_copy(self._memory(address, ctypes.sizeof(_MessageHeader))) + if not 1 <= header.vector_count <= 4 or header.control_length > socket.CMSG_SPACE(MAX_RESOURCES * array.array("i").itemsize) or header.flags & (socket.MSG_TRUNC | socket.MSG_CTRUNC): + raise AssertionError("native protocol message vectors or ancillary data exceed their bound") + vectors = self._memory(int(header.vectors or 0), ctypes.sizeof(_Iovec) * header.vector_count) + data = bytearray() + remaining = received_length + for index in range(header.vector_count): + item = _Iovec.from_buffer_copy(vectors[index * ctypes.sizeof(_Iovec) : (index + 1) * ctypes.sizeof(_Iovec)]) + size = int(item.length) if remaining is None else min(int(item.length), remaining) + if size < 0 or size > MAX_MESSAGE_BYTES + 1 or len(data) + size > MAX_MESSAGE_BYTES + 1: + raise AssertionError("native protocol message bytes exceed their bound") + if size: + data.extend(self._memory(int(item.base or 0), size)) + if remaining is not None: + remaining -= size + if remaining == 0: + break + if remaining not in (None, 0): + raise AssertionError("actual recvmsg result exceeds its real iovec capacity") + control = self._memory(int(header.control or 0), int(header.control_length)) if header.control_length else b"" + return bytes(data), _decode_rights(control) + + def _selected(self, encoded: bytes) -> bool: + if not encoded.endswith(b"\n") or encoded.count(b"\n") != 1 or len(encoded) > MAX_MESSAGE_BYTES: + return False + try: + value = json.loads(encoded) + except (UnicodeError, ValueError): + return False + return isinstance(value, dict) and value.get("schema") == self.schema and all(value.get(name) == selected for name, selected in self.binding.items()) + + def _observe_call(self) -> bool: + info = _Information() + length = self._ptrace(_PTRACE_GET_SYSCALL_INFO, ctypes.sizeof(info), ctypes.cast(ctypes.byref(info), ctypes.c_void_p)) + if length < 24 or info.architecture != _AUDIT_ARCH_X86_64: + raise AssertionError("actual native protocol syscall information is unavailable") + if info.operation == 1: + self._message_call = None + call = info.call.entry + number, descriptor, address = int(call.number), int(call.arguments[0]), int(call.arguments[1]) + if self.direction == "receive" and number == _SYS_RECVMSG: + self._message_call = (number, descriptor, address, 0) + elif self.direction == "send" and number in {_SYS_SENDMSG, _SYS_SENDTO}: + if number == _SYS_SENDMSG: + encoded, descriptors = self._message(address, received_length=None) + else: + size = int(call.arguments[2]) + if not 0 < size <= MAX_MESSAGE_BYTES: + return False + encoded, descriptors = self._memory(address, size), () + if self._selected(encoded): + if self.before_send is not None: + if self._before_called: + raise AssertionError("the actual selected protocol send was attempted twice") + self.before_send() + self._before_called = True + self._message_call = (number, descriptor, address, len(encoded)) + self._selected_message = (encoded, descriptors, descriptor, None) + return self.message_when == "before" + elif info.operation == 2 and self._message_call is not None: + number, descriptor, address, expected_size = self._message_call + self._message_call = None + result = int(info.call.exit.result) + if info.call.exit.is_error or result <= 0: + return False + if self.direction == "receive": + encoded, descriptors = self._message(address, received_length=result) + if not self._selected(encoded): + return False + self._selected_message = (encoded, descriptors, descriptor, result) + return True + if self._selected_message is None or result != expected_size: + raise AssertionError("the actual selected protocol send was incomplete") + encoded, descriptors, _, _ = self._selected_message + self._selected_message = (encoded, descriptors, descriptor, result) + return True + return False + + def _record(self) -> None: + self._require_lifetime() + if self._selected_message is None: + raise AssertionError("the actual selected protocol message is absent") + encoded, descriptors, descriptor, result = self._selected_message + path = Path(f"/proc/{self.issuer.pid}/fd/{descriptor}") + info = path.stat() + if not stat.S_ISSOCK(info.st_mode): + raise AssertionError("the actual protocol descriptor is not a kernel socket") + if self.schema == OFFER_SCHEMA and not descriptors or self.schema != OFFER_SCHEMA and descriptors: + raise AssertionError("the actual protocol ancillary descriptors differ from the expected message class") + objects = [] + for received in descriptors: + item = Path(f"/proc/{self.issuer.pid}/fd/{received}") + actual = item.stat() + objects.append((received, actual.st_dev, actual.st_ino, os.readlink(item))) + self._observation = PeerMessageObservation(self.issuer.pid, self.start_token, self.boot_id, self.schema, self.direction, self.message_when, descriptor, info.st_ino, hashlib.sha256(encoded).hexdigest(), len(encoded), result, descriptors, tuple(objects)) + logger.bind(native_peer_schema=self.schema, native_peer_direction=self.direction, native_peer_boundary=self.message_when, process_id=self.issuer.pid).info("Actual production protocol process stopped at its selected kernel message boundary") + self._hit.set() + + def wait(self, timeout: float = 900) -> PeerMessageObservation: + result = self.wait_observation(timeout) + if not isinstance(result, PeerMessageObservation): + raise AssertionError("the native peer barrier did not observe its actual message") + return result + + +def _descriptor_snapshot(pid: int) -> tuple[tuple[int, int, int, str], ...]: + result = [] + for path in sorted(Path(f"/proc/{pid}/fd").iterdir(), key=lambda item: int(item.name)): + info = path.stat() + result.append((int(path.name), info.st_dev, info.st_ino, os.readlink(path))) + return tuple(result) + + +def _assert_no_target_admission(context: NativeNegativeContext, *, running_pid: int | None = None) -> None: + state = Path(context.enrollment.target.owner.path).parent + config = BootstrapConfig.read(Path(context.enrollment.target.bootstrap_config.path)) + journals = [item for item in context.enrollment.target.mounts if item.destination == str(config.journal_root) and item.writable] + if len(journals) != 1 or set(os.listdir(state)) - {".supervisor-owner.lock", "logs"} or os.listdir(journals[0].source): + raise AssertionError("the prelaunch target created a native consumer claim or operation") + if running_pid is not None and Path(f"/proc/{running_pid}/task/{running_pid}/children").read_text().strip(): + raise AssertionError("the actual prelaunch target launched a child") + for path in (state / "logs").glob("recovery-epoch-bootstrap*.log"): + if "Recovery custody acknowledged" in path.read_text(): + raise AssertionError("the actual consumer accepted an unauthorized current grant") + + +def _assert_received_objects(context: NativeNegativeContext, observation: PeerMessageObservation, issuer_pid: int) -> None: + expected = {(item.device, item.inode): item.kind for item in _consumer_configuration(context).transferred_resources} + if len(observation.descriptor_objects) != len(expected) or {(device, inode) for _, device, inode, _ in observation.descriptor_objects} != set(expected): + raise AssertionError("actual recvmsg did not install the exact complete original object set") + for descriptor, device, inode, path in observation.descriptor_objects: + if expected[device, inode] == "network-namespace": + if path != f"net:[{inode}]": + raise AssertionError("actual recvmsg namespace object differs") + else: + lock = inspect_kernel_lock(observation.process_id, descriptor, device, inode) + if lock is None or lock.mode != "WRITE" or lock.pid != issuer_pid: + raise AssertionError("actual recvmsg did not retain the issuer's exclusive OFD") + + +def _stop_target_after_failure(context: NativeNegativeContext) -> None: + state = _target_state(context) + if state.get("Running") is True: + target = DockerTraceProcess(context) + try: + target.kill() + target.wait(30) + finally: + target.close() + + +def _run_peer_case(context: NativeNegativeContext, case: str) -> dict[str, object]: + require_native_context(context) + if case not in REQUIRED_PEER_CASES: + raise ValueError("a reviewed production peer refusal is required") + if _target_state(context).get("Running") is not False or PEER_SOCKET.exists() or PEER_SOCKET.is_symlink(): + raise AssertionError("a previous target lifetime or issuer endpoint remains; never reset it to retry") + epoch_before = _epoch_snapshot(context.epoch_root) + observations: dict[str, object] = {"case": case} + log = context.fixture_root / ("native-peer-" + case + ".log") + issuer = None + host_barrier = None + target_barrier = None + target: DockerTraceProcess | None = None + replacement: DockerTraceProcess | None = None + paused_snapshot: tuple[tuple[int, int, int, str], ...] | None = None + with ExitStack() as held: + output = held.enter_context(log.open("xb")) + try: + issuer = subprocess.Popen([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], stdout=output, stderr=subprocess.STDOUT, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + + def stop_receiver_before_offer() -> None: + nonlocal target, paused_snapshot + target = DockerTraceProcess(context) + target.pause() + paused_snapshot = _descriptor_snapshot(target.pid) + _assert_no_target_admission(context, running_pid=target.pid) + + receiving_offer = case == "received-rights-without-receipt-refused" + challenge = case in {"changed-target-lifetime-refused", "foreign-acquirer-sh-refused"} + host_barrier = KernelPeerMessageBarrier( + issuer, + context=context, + schema=CHALLENGE_SCHEMA if challenge else OFFER_SCHEMA, + direction="receive" if challenge else "send", + when="before" if receiving_offer else "after", + before_send=stop_receiver_before_offer if case == "queued-rights-without-receipt-refused" else None, + ) + host_barrier.start() + host_barrier.resume_initial() + _wait(lambda: PEER_SOCKET.exists() and "Fixed recovery issuer endpoint is listening" in log.read_text(), "the actual fixed issuer endpoint is listening", timeout=120) + _docker("start", context.enrollment.target.management_id) + observation = host_barrier.wait(timeout=600) + observations["issuer_syscall"] = asdict(observation) + assert_unreserved_epoch(context.epoch_root) + if case == "queued-rights-without-receipt-refused": + if target is None or paused_snapshot is None or _descriptor_snapshot(target.pid) != paused_snapshot or observation.syscall_result != observation.bytes_observed: + raise AssertionError("the successful native send did not precede actual receiver custody") + observations["receiver_before_and_after_enqueue"] = paused_snapshot + _assert_no_target_admission(context, running_pid=target.pid) + target.kill() + target.wait(30) + host_barrier.release() + elif receiving_offer: + target = DockerTraceProcess(context) + target_barrier = KernelPeerMessageBarrier(target, context=context, schema=OFFER_SCHEMA, direction="receive") + target_barrier.start() + target_barrier.resume_initial() + host_barrier.release() + received = target_barrier.wait(timeout=90) + _assert_received_objects(context, received, issuer.pid) + observations["receiver_syscall"] = asdict(received) + assert_unreserved_epoch(context.epoch_root) + _assert_no_target_admission(context, running_pid=target.pid) + target_barrier.kill() + elif case == "changed-target-lifetime-refused": + target = DockerTraceProcess(context) + observations["original_target_lifetime"] = {"pid": target.pid, "start_token": target.start_token} + target.kill() + target.wait(30) + _docker("start", context.enrollment.target.management_id) + replacement = DockerTraceProcess(context) + if (replacement.pid, replacement.start_token) == (target.pid, target.start_token): + raise AssertionError("the actual target restart did not create a new lifetime") + observations["replacement_target_lifetime"] = {"pid": replacement.pid, "start_token": replacement.start_token} + _assert_no_target_admission(context, running_pid=replacement.pid) + host_barrier.release() + else: + target = DockerTraceProcess(context) + original = next(item for item in _consumer_configuration(context).original_fences if item.name.startswith("owner-")) + observations["foreign_original_custody"] = held.enter_context(_foreign_acquirer(target.pid, original.device, original.inode, "READ")) + host_barrier.release() + # Trace ownership has ended before ordinary Popen wait/poll. + issuer.wait(timeout=120) + if issuer.returncode != 2: + raise AssertionError("the actual issuer did not refuse the incomplete native custody") + _wait(lambda: _target_state(context).get("Running") is False, "the actual refused consumer exits", timeout=90) + _assert_no_target_admission(context) + if PEER_SOCKET.exists() or PEER_SOCKET.is_symlink(): + raise AssertionError("the original refusing issuer left its endpoint; never unlink it to retry") + if _epoch_snapshot(context.epoch_root) != epoch_before or "One consumed recovery epoch grant send completed" in log.read_text(): + raise AssertionError("incomplete native peer custody published or dispatched an epoch") + observations.update(issuer_returncode=issuer.returncode, epoch_unreserved=True, issuer_endpoint_removed_by_original=True, consumer_grant_accepted=False, no_child_or_admission=True) + return observations + finally: + if target_barrier is not None: + target_barrier.close() + if host_barrier is not None: + host_barrier.close() + if issuer is not None and issuer.poll() is None: + issuer.kill() + issuer.wait(timeout=30) + _stop_target_after_failure(context) + for process in (replacement, target): + if process is not None: + process.close() + + +def run_native_peer_cases(context: NativeNegativeContext, preserved_snapshot: Callable[[], object]) -> list[str]: + """Require actual peer refusal before every original epoch publication.""" + require_native_context(context) + observations = [] + for case in sorted(REQUIRED_PEER_CASES): + preserved = preserved_snapshot() + observation = _run_peer_case(context, case) + if preserved_snapshot() != preserved: + raise AssertionError("actual peer refusal changed original records, OAuth or unrelated workers") + observations.append(observation) + logger.bind(native_peer_case=case, epoch_unreserved=True, consumer_grant_accepted=False).info("Actual production peer refusal preserved original state") + destination = context.fixture_root / "native-peer-observations.json" + descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(canonical({"schema": "actual-recovery-native-peer-faults-v1", "cases": observations})) + stream.flush() + os.fsync(stream.fileno()) + return [str(item["case"]) for item in observations] diff --git a/scripts/recovery_vm/README.md b/scripts/recovery_vm/README.md index ae8d95949..8d1ecc908 100644 --- a/scripts/recovery_vm/README.md +++ b/scripts/recovery_vm/README.md @@ -21,6 +21,13 @@ The outer QEMU container has a read-only root, no capabilities, no new privilege disk and bounded output are mounted. Each scenario gets a new directory and new 32 GiB sparse guest disk; an existing disk or original epoch is never reset. +Preparation also binds the actual source Docker image store (`overlay2` or the +`overlayfs` containerd snapshotter). The fresh guest configures the same store +before its first daemon start and verifies both imported images retain their +exact approved IDs. Importing between these stores can change the meaning of +Docker's image ID; a changed ID is a preparation failure, not a reason to select +another image or weaken the production source checks. + ## Reproduce a reviewed scenario Use a clean, committed checkout with the locked test dependencies installed. The @@ -49,9 +56,13 @@ that would discard this selection. The VM-only authorization marker binds the actual new machine, boot and engine. The driver verifies it before provisioning any management fixture. It refuses a populated engine or earlier fixture roots. -`SCENARIOS` and `required_cases()` in -`scripts/test_management_recovery_epoch_host.py` define the finite scenario -oracle. All before/after publication cases run on independent fresh VMs. They +`ACCEPTANCE_SCENARIOS` and `required_cases()` in +`scripts/test_management_recovery_epoch_host.py` define the complete finite +scenario oracle. CI derives its matrix directly from this list and requires +every matrix job to pass. The two `kill-after-*` debug aliases select boundaries +already present in the matrix and do not add independent cases. All before/after +publication cases and all ten mutations after the original intent head run on +independent fresh VMs. They use real ptrace syscall stops around the actual exclusive record/head creation and successful file/directory fsync, with a pinned SIGKILL and no syscall-result replacement. The native process oracles also inspect real peer custody, original @@ -59,6 +70,18 @@ store preservation, unrelated worker lifetimes and effect requests, and absence of a replacement child/admission. Component guard tests cannot substitute for these executions. +The `grant` scenario first requires all four actual protocol refusals implemented +in `scripts/recovery_epoch_native_peer.py`: queued rights without a receipt, +received rights without a receipt, a replaced target lifetime, and an SH lock +whose acquiring process differs from the target. Each must preserve an entirely +unreserved epoch before a later independent invocation can grant. Its successful +path also holds the original issuer at the durable reservation head while an +overlapping contender is refused. The separate `lost-reply` scenario requires an +actual successful grant send while the receiver is stopped, then verifies the +issuer's pidfd is dead before resuming the receiver; consumption must remain and +the receiver must refuse current custody. The companion helper tests validate +the fixture's native message parsing and guards; they are not VM acceptance. + The guest read credential is deleted on exit. The outer input credential must also be deleted after the run; the CI workflow does this even on failure. Artifacts contain the result, bounded diagnostics and guest/serial logs, not the diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh index 4621aa925..542df0858 100644 --- a/scripts/recovery_vm/guest.sh +++ b/scripts/recovery_vm/guest.sh @@ -11,6 +11,12 @@ finish() { trap - EXIT set +e rm -f /run/recovery-fixture-source-token + { + systemctl cat cloud-final.service cloud-config.service cloud-init.service cloud-init-local.service + systemctl show cloud-final.service cloud-config.service cloud-init.service cloud-init-local.service -p ExecStart -p ActiveState -p UnitFileState + docker version + docker compose version + } > "$output/guest-platform.txt" 2>&1 printf '%s\n' "$result" | tee "$output/guest-exit-code" > /dev/null sync systemctl poweroff --no-block @@ -33,9 +39,16 @@ for name, digest in manifest["inputs_sha256"].items(): raise ValueError("Recovery fixture input changed: " + name) PY export DEBIAN_FRONTEND=noninteractive +# Match the source store before the new daemon starts; cross-store imports can +# change Docker's image IDs even when their layers are identical. +mkdir -p /etc/docker +test ! -e /etc/docker/daemon.json +test ! -L /etc/docker/daemon.json +install -m 600 "$input/docker-daemon.json" /etc/docker/daemon.json apt-get update apt-get install -y --no-install-recommends ca-certificates cron curl docker.io docker-compose-v2 git python3-venv systemctl start docker +docker info --format 'Actual guest image store: {{.Driver}} {{json .DriverStatus}}' python3 -m venv /opt/fixture-tools /opt/fixture-tools/bin/pip install uv==0.13.6 mkdir -m 700 /opt/recovery-fixture /opt/reviewed-source @@ -66,6 +79,11 @@ git -C /opt/reviewed-source fetch --no-tags origin "${selected[3]}" git -C /opt/reviewed-source checkout --detach FETCH_HEAD test "$(git -C /opt/reviewed-source rev-parse --is-shallow-repository)" = false docker image load --input "$input/images.tar" +docker image ls --no-trunc +for selected_image in "${selected[1]}" "${selected[2]}"; do + printf 'Verifying imported fixture image %s\n' "$selected_image" + test "$(docker image inspect --format '{{.Id}}' "$selected_image")" = "$selected_image" +done python3 - "$input" <<'PY' import json import os diff --git a/scripts/recovery_vm/prepare.sh b/scripts/recovery_vm/prepare.sh index 654cf33cd..58613295d 100644 --- a/scripts/recovery_vm/prepare.sh +++ b/scripts/recovery_vm/prepare.sh @@ -42,14 +42,34 @@ fi for selected in "$manager_image" "$proxy_image"; do test "$(docker image inspect --format '{{.Id}}' "$selected")" = "$selected" done +fixture_driver=$(docker info --format '{{.Driver}}') +fixture_driver_status=$(docker info --format '{{json .DriverStatus}}') +if [[ "$fixture_driver" == overlayfs && "$fixture_driver_status" == *'io.containerd.snapshotter.v1'* ]]; then + fixture_image_store=containerd +elif [[ "$fixture_driver" == overlay2 && "$fixture_driver_status" != *'io.containerd.snapshotter.v1'* ]]; then + fixture_image_store=overlay2 +else + echo 'The recovery VM requires a supported, explicitly matched Docker image store.' >&2 + exit 2 +fi mkdir -m 700 "$fixture_root" "$fixture_root/input" "$fixture_root/output" "$fixture_root/disk" exec > >(tee -a "$fixture_root/output/preparation.log") 2>&1 +printf 'Exporting exact fixture IDs from Docker image store %s.\n' "$fixture_image_store" +if [[ "$fixture_image_store" == overlay2 ]]; then + tee "$fixture_root/input/docker-daemon.json" > /dev/null <<'JSON' +{"features":{"containerd-snapshotter":false},"storage-driver":"overlay2"} +JSON +else + tee "$fixture_root/input/docker-daemon.json" > /dev/null <<'JSON' +{"features":{"containerd-snapshotter":true},"storage-driver":"overlayfs"} +JSON +fi git -C "$repo_root" archive --format=tar.gz HEAD | tee "$fixture_root/input/source.tar.gz" > /dev/null docker image save "$manager_image" "$proxy_image" | tee "$fixture_root/input/images.tar" > /dev/null cp "$tooling_root/guest.sh" "$fixture_root/input/guest.sh" install -m 600 "$source_token_file" "$fixture_root/input/source-token" fixture_commit=$(git -C "$repo_root" rev-parse HEAD) -python3 - "$fixture_root" "$fixture_authorization" "$fixture_commit" "$manager_image" "$proxy_image" "$fixture_scenario" "$fixture_accelerator" <<'PY' +python3 - "$fixture_root" "$fixture_authorization" "$fixture_commit" "$manager_image" "$proxy_image" "$fixture_scenario" "$fixture_accelerator" "$fixture_image_store" <<'PY' import hashlib import json import pathlib @@ -57,11 +77,11 @@ import sys root = pathlib.Path(sys.argv[1]) inputs = root / "input" -manifest = dict(zip(("authorization_id", "commit", "manager_image", "proxy_image", "scenario", "accelerator"), sys.argv[2:])) +manifest = dict(zip(("authorization_id", "commit", "manager_image", "proxy_image", "scenario", "accelerator", "image_store"), sys.argv[2:])) manifest["schema"] = "auto-coder-recovery-vm-input-v1" manifest["ready_source_sha"] = "e742338b3f311e6d30a22a8f5341b8072513d813" manifest["inputs_sha256"] = {} -for name in ("source.tar.gz", "images.tar", "guest.sh"): +for name in ("source.tar.gz", "images.tar", "guest.sh", "docker-daemon.json"): with (inputs / name).open("rb") as stream: manifest["inputs_sha256"][name] = hashlib.file_digest(stream, "sha256").hexdigest() (inputs / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 0bcb78e85..49a345515 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -31,6 +31,7 @@ from scripts.recovery_epoch_containment_fixture import CapturedGeneration, capture_generation, enroll_recovery_scope, selected_file, start_watcher, target_from_container from scripts.recovery_epoch_fault_tracer import BOUNDARY_SCENARIOS, PUBLICATIONS, KernelPublicationBarrier, PublicationObservation, parse_boundary from scripts.recovery_epoch_native_negatives import REQUIRED_NEGATIVE_CASES, REQUIRED_REPORT_CASES, NativeNegativeContext, _replacement, assert_unreserved_epoch, run_native_negatives, verify_native_containment_reports +from scripts.recovery_epoch_native_peer import LATE_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, mutate_after_intent, pause_target_before_grant, run_native_peer_cases from scripts.test_management_containment_closed_host import InhibitedStockRoutes, WatcherFixture, require_disposable_host from scripts.test_management_containment_enrollment_host import DisposableSystemdRoutes, _private_directory, _write_private @@ -56,7 +57,8 @@ RESULT_SCHEMA = "auto-coder-native-recovery-epoch-result-v1" ORIGINAL_NEGATIVE_CASES = frozenset({"original-journal-absence-refused", "changed-installed-ready-refused", "changed-target-management-config-refused"}) LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused"}) -SCENARIOS = ("grant", "kill-after-intent", "kill-after-consume", "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS) +ACCEPTANCE_SCENARIOS = ("grant", "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS, *LATE_MUTATION_SCENARIOS) +SCENARIOS = (*ACCEPTANCE_SCENARIOS, "kill-after-intent", "kill-after-consume") REQUIRED_CASES = frozenset( { "affirmative-original-genesis", @@ -81,7 +83,7 @@ def required_cases(scenario: str) -> frozenset[str]: if scenario not in SCENARIOS: raise ValueError("unsupported native recovery fixture scenario") if scenario == "grant": - return REQUIRED_CASES | REQUIRED_NEGATIVE_CASES | REQUIRED_REPORT_CASES | ORIGINAL_NEGATIVE_CASES | LINEAGE_REPORT_CASES + return REQUIRED_CASES | REQUIRED_NEGATIVE_CASES | REQUIRED_REPORT_CASES | REQUIRED_PEER_CASES | ORIGINAL_NEGATIVE_CASES | LINEAGE_REPORT_CASES return (REQUIRED_CASES - {"complete-native-epoch-granted", "old-owner-and-controller-contenders-refused", "issuer-death-ended-current-custody", "consumed-epoch-replay-refused", "reservation-barrier-contender-refused"}) | {"native-" + scenario, "interrupted-epoch-replay-refused"} @@ -170,6 +172,21 @@ def _wait(predicate: Callable[[], bool], description: str, *, timeout: float = 9 time.sleep(0.05) +def _command_failure(error: BaseException) -> dict[str, object] | None: + """Retain bounded child stderr/stdout without exporting command inputs.""" + if not isinstance(error, (subprocess.CalledProcessError, subprocess.TimeoutExpired)): + return None + lines = [] + for value in (error.output, error.stderr): + if isinstance(value, bytes): + value = value.decode(errors="replace") + if isinstance(value, str): + # The prefix keeps plain Compose diagnostics in the same existing + # credential/header/body filter used for all exported fixture logs. + lines.extend(diagnostic_log_lines("\n".join("RuntimeError: " + line for line in value.splitlines()[-60:]))) + return {"type": type(error).__name__, "returncode": getattr(error, "returncode", None), "diagnostics": lines[-60:]} + + def _checkpoint_stopped(generation: Generation) -> None: """The stopped fixture producer settles its own stores before inspection.""" for directory in (generation.path("state"), generation.path("journal")): @@ -396,7 +413,7 @@ def _target( published_port = reservation.getsockname()[1] if published_port == chain.port: raise AssertionError("new target endpoint must differ from every old endpoint") - command = ["create", "--name", name, "--read-only", "--network", "bridge", "--publish", f"127.0.0.1:{published_port}:8080", "--env", "PYTHONDONTWRITEBYTECODE=1"] + command = ["create", "--name", name, "--read-only", "--workdir", "/workspace", "--network", "bridge", "--publish", f"127.0.0.1:{published_port}:8080", "--env", "PYTHONDONTWRITEBYTECODE=1"] for source, destination, writable in mounts: command.extend(("--mount", f"type=bind,src={source},dst={destination}" + ("" if writable else ",readonly"))) command.extend(("--entrypoint", "python", chain.image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json", "--recovery-epoch")) @@ -505,6 +522,13 @@ def validate_result(value: dict[str, object]) -> None: assert isinstance(value.get("durable"), dict) if scenario == "grant": assert set(value["durable"]["record_sha256"]) == {"reserved", "intent", "consumed", "receipt"} + peer = value.get("peer_observations", {}) + assert peer.get("schema") == "actual-recovery-native-peer-faults-v1" + assert {item.get("case") for item in peer.get("cases", ())} == REQUIRED_PEER_CASES + for item in peer["cases"]: + assert item.get("issuer_returncode") == 2 and item.get("epoch_unreserved") is True and item.get("issuer_endpoint_removed_by_original") is True + assert item.get("consumer_grant_accepted") is False and item.get("no_child_or_admission") is True + assert item.get("issuer_syscall", {}).get("bytes_observed", 0) > 0 elif scenario in BOUNDARY_SCENARIOS: publication, when = parse_boundary(scenario) assert value["durable"]["publication"] == publication and value["durable"]["when"] == when @@ -513,40 +537,58 @@ def validate_result(value: dict[str, object]) -> None: elif scenario == "retarget-after-reservation": assert value["durable"]["phase"] == "reserved" and value["durable"]["consumer_grant_acknowledged"] is False assert value["durable"]["inventory_sha256"] != value["durable"]["retargeted_inventory_sha256"] + elif scenario in LATE_MUTATION_SCENARIOS: + assert value["durable"]["phase"] == "intent" and value["durable"]["consumer_grant_acknowledged"] is False and value["durable"]["grant_send_completed"] is False + assert value["durable"]["mutation"]["scenario"] == scenario and value["durable"]["mutation"]["boundary"] == "after-intent-head" + assert value.get("preservation_baseline") == "after-deliberate-fixture-mutation" + assert value["durable"]["mutation"]["preservation_baseline"] == value["preservation_baseline"] + assert value["preserved_originals_sha256"] == value["durable"]["originals_after_injection_sha256"] else: assert value["durable"]["phase"] == ("intent" if scenario == "kill-after-intent" else "consumed" if scenario == "kill-after-consume" else "receipt") assert value["durable"]["consumer_grant_acknowledged"] is False + assert value["durable"]["grant_send_completed"] is (scenario == "lost-reply") + if scenario == "lost-reply": + assert value["durable"]["receiver_paused_before_send"] is True and value["durable"]["issuer_died_before_receiver_resumed"] is True + + +def _selected_boundary(scenario: str) -> tuple[str, str]: + if scenario in BOUNDARY_SCENARIOS: + return parse_boundary(scenario) + if scenario in LATE_MUTATION_SCENARIOS or scenario == "kill-after-intent": + return "intent-head.json", "after" + if scenario == "kill-after-consume": + return "consumed-head.json", "after" + if scenario == "lost-reply": + return "receipt-head.json", "after" + if scenario in {"grant", "retarget-after-reservation"}: + return "reserved-head.json", "after" + raise ValueError("unsupported native recovery publication barrier") -def _interrupted_issuer(scenario: str, issuer: subprocess.Popen, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: - """Signal real post-fsync publications; reject a race beyond the boundary.""" - phase = "intent" if scenario == "kill-after-intent" else "consumed" if scenario == "kill-after-consume" else "receipt" - selected_pid = issuer.pid if scenario != "lost-reply" else int(Chain.docker("inspect", "--format", "{{.State.Pid}}", target.generation.container)) - descriptor = os.pidfd_open(selected_pid, 0) +def _lost_reply(context: NativeNegativeContext, barrier: KernelPublicationBarrier, issuer: subprocess.Popen, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: + """Observe successful delivery to the socket before the pinned issuer dies.""" + original_epoch = _snapshot((EPOCH_ROOT,)) + descriptor = os.pidfd_open(issuer.pid, 0) try: - _wait(lambda: f"Recovery epoch {phase} durably recorded" in issuer_log.read_text(), "actual durable " + phase + " publication", timeout=600, process=issuer) - signal.pidfd_send_signal(descriptor, signal.SIGKILL) + with pause_target_before_grant(context, issuer.pid) as observation: + assert "Recovery custody acknowledged" not in consumer_log.read_text() + barrier.release() + _wait(lambda: "One consumed recovery epoch grant send completed" in issuer_log.read_text(), "actual grant send while its receiver remains kernel-stopped", timeout=1800, process=issuer) + assert "Recovery custody acknowledged" not in consumer_log.read_text(), "the stopped receiver observed the grant" + assert _snapshot((EPOCH_ROOT,)) == original_epoch + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + assert issuer.wait(timeout=30) == -signal.SIGKILL + assert "Recovery custody acknowledged" not in consumer_log.read_text() finally: os.close(descriptor) - if scenario == "lost-reply": - issuer.wait(timeout=120) - assert issuer.returncode == 2, "issuer accepted a dead consumer after durable consumption" - else: - issuer.wait(timeout=30) - assert issuer.returncode == -signal.SIGKILL - _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "interrupted real peer lifetime ends", timeout=90) + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "queued grant cannot survive the actual issuer lifetime", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" _no_target_child(target.generation, running=False) - assert "Recovery custody acknowledged" not in consumer_log.read_text(), "signal raced beyond the required pregrant boundary" - assert "One consumed recovery epoch grant send completed" not in issuer_log.read_text(), "signal raced beyond actual grant dispatch" - encoded = (EPOCH_ROOT / (phase + ".json")).read_bytes() - head = json.loads((EPOCH_ROOT / (phase + "-head.json")).read_bytes()) - assert head["phase"] == phase and head["record_sha256"] == hashlib.sha256(encoded).hexdigest() - if scenario == "kill-after-intent": - assert not (EPOCH_ROOT / "consumed.json").exists(), "intent interruption raced beyond consumption" - before = _snapshot((EPOCH_ROOT,)) + assert "Recovery custody acknowledged" not in consumer_log.read_text() + assert _snapshot((EPOCH_ROOT,)) == original_epoch replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) - assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == before, "an interrupted original epoch was repaired or reissued" - return {"phase": phase, "record_sha256": hashlib.sha256(encoded).hexdigest(), "consumer_grant_acknowledged": False, "grant_send_completed": False} + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == original_epoch, "a consumed epoch was reissued after its reply was lost" + return {"phase": "receipt", "consumer_grant_acknowledged": False, "grant_send_completed": True, "receiver_paused_before_send": True, "issuer_died_before_receiver_resumed": True, "native_observation": observation, "epoch_sha256": original_epoch} def _publication_interruption(scenario: str, observed: PublicationObservation, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: @@ -679,6 +721,48 @@ def _retarget_reserved(barrier: KernelPublicationBarrier, issuer: subprocess.Pop return {"phase": "reserved", "consumer_grant_acknowledged": False, "inventory_sha256": original_digest, "retargeted_inventory_sha256": _digest(inventory), "retained_publications": ["reserved.json", "reserved-head.json"]} +def _late_mutation( + context: NativeNegativeContext, + scenario: str, + barrier: KernelPublicationBarrier, + issuer: subprocess.Popen, + issuer_log: Path, + consumer_log: Path, + target: PreparedTarget, + retained_roots: tuple[Path, ...], + check_workers: Callable[[], None], +) -> tuple[dict[str, object], str]: + """Require an actual late refusal while retaining the injected uncertainty.""" + original = _snapshot(retained_roots) + with mutate_after_intent(context, scenario, issuer.pid) as observation: + injected = _snapshot(retained_roots) + epoch = _snapshot((EPOCH_ROOT,)) + check_workers() + barrier.release() + assert issuer.wait(timeout=1800) == 2, "the actual issuer did not refuse the changed post-intent authority" + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "late refusal ends the exact current consumer lifetime", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() and "One consumed recovery epoch grant send completed" not in issuer_log.read_text() + assert _snapshot((EPOCH_ROOT,)) == epoch, "late mutation reached consumption or rewrote original epoch evidence" + assert _snapshot(retained_roots) == injected, "production refusal changed the deliberately injected original state" + check_workers() + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == epoch, "the uncertain original intent was repaired or reissued" + assert _snapshot(retained_roots) == injected + check_workers() + result = { + "phase": "intent", + "consumer_grant_acknowledged": False, + "grant_send_completed": False, + "mutation": observation, + "originals_before_injection_sha256": original, + "originals_after_injection_sha256": injected, + "epoch_sha256": epoch, + } + return result, injected + + def run_fixture(*, authorization_id: str, source_repository: Path, source_token_file: Path, source_sha: str, uv: Path, uv_version: str, output: Path, scenario: str = "grant") -> dict[str, object]: required_cases(scenario) if not re.fullmatch(r"[0-9a-f]{40}", source_sha) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", uv_version): @@ -783,6 +867,8 @@ def preservation() -> object: cases.extend(LINEAGE_REPORT_CASES) cases.extend(run_native_negatives(native_context, lambda label: _issuer_refusal(native_context, target, label), preservation)) cases.extend(_original_negatives(native_context, target, chain, preservation)) + cases.extend(run_native_peer_cases(native_context, preservation)) + result["peer_observations"] = json.loads((root / "native-peer-observations.json").read_bytes()) # Only the intentional reversible fixture injections changed path # metadata. This fresh strict baseline precedes actual issuance. preserved = _snapshot(retained_roots) @@ -793,15 +879,11 @@ def preservation() -> object: contender = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) assert contender.returncode == 2 and not (EPOCH_ROOT / "reserved.json").exists(), "a second issuer acquired the same current epoch" cases.append("independent-issuer-contender-refused") - # A real SIGSTOP barrier lets the exact target execute its installed - # interpreter probe before T takes its current process observations. - if scenario in BOUNDARY_SCENARIOS or scenario in ("grant", "retarget-after-reservation"): - publication, when = parse_boundary(scenario) if scenario in BOUNDARY_SCENARIOS else ("reserved-head.json", "after") - barrier = KernelPublicationBarrier(issuer, root=EPOCH_ROOT, name=publication, when=when) - barrier.start() - else: - issuer.send_signal(signal.SIGSTOP) - _wait(lambda: Path(f"/proc/{issuer.pid}/stat").read_text().rsplit(")", 1)[1].split()[0] == "T", "issuer kernel stop barrier", process=issuer) + # The actual ptrace stop lets the target finish its interpreter probe + # before T samples current tasks, then observes the chosen publication. + publication, when = _selected_boundary(scenario) + barrier = KernelPublicationBarrier(issuer, root=EPOCH_ROOT, name=publication, when=when) + barrier.start() chain.docker("start", target.generation.container) probe = json.loads(chain.docker("exec", target.generation.container, target.receipt.interpreter_path, "-I", "-B", "-m", "auto_coder.build_provenance", timeout=60).splitlines()[-1]) assert probe["source_revision"]["available"] is True and probe["source_revision"]["value"] == source_sha @@ -815,29 +897,27 @@ def preservation() -> object: workers.preserved(*worker_before) consumer_log = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" result["source_proof"] = {name: getattr(target.receipt, name) for name in ("source_sha", "tested_source_ref", "tested_source_object", "test_run_id", "test_run_attempt", "lock_sha256", "distribution_version", "receipt_identity")} - if barrier is not None: - barrier.resume_initial() - observed_boundary = barrier.wait(timeout=1800) - if scenario == "grant": - _reserved_contender() - cases.append("reservation-barrier-contender-refused") - result["reservation_barrier"] = asdict(observed_boundary) - barrier.release() - else: - if scenario == "retarget-after-reservation": - result["durable"] = _retarget_reserved(barrier, issuer, issuer_log, consumer_log, target) - else: - barrier.kill() - result["durable"] = _publication_interruption(scenario, observed_boundary, issuer_log, consumer_log, target) - cases.extend(("native-" + scenario, "interrupted-epoch-replay-refused")) - assert _snapshot(retained_roots) == preserved - workers.preserved(*worker_before) - _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) - return result + barrier.resume_initial() + observed_boundary = barrier.wait(timeout=1800) + if scenario == "grant": + _reserved_contender() + cases.append("reservation-barrier-contender-refused") + result["reservation_barrier"] = asdict(observed_boundary) + barrier.release() else: - issuer.send_signal(signal.SIGCONT) - if scenario != "grant": - result["durable"] = _interrupted_issuer(scenario, issuer, issuer_log, consumer_log, target) + if scenario == "retarget-after-reservation": + result["durable"] = _retarget_reserved(barrier, issuer, issuer_log, consumer_log, target) + elif scenario == "lost-reply": + result["durable"] = _lost_reply(native_context, barrier, issuer, issuer_log, consumer_log, target) + elif scenario in LATE_MUTATION_SCENARIOS: + result["durable"], preserved = _late_mutation(native_context, scenario, barrier, issuer, issuer_log, consumer_log, target, retained_roots, lambda: workers.preserved(*worker_before)) + result["preservation_baseline"] = "after-deliberate-fixture-mutation" + else: + barrier.kill() + actual = scenario if scenario in BOUNDARY_SCENARIOS else "after-" + publication.removesuffix(".json") + result["durable"] = _publication_interruption(actual, observed_boundary, issuer_log, consumer_log, target) + if scenario in {"kill-after-intent", "kill-after-consume"}: + result["durable"]["phase"] = "intent" if scenario == "kill-after-intent" else "consumed" cases.extend(("native-" + scenario, "interrupted-epoch-replay-refused")) assert _snapshot(retained_roots) == preserved workers.preserved(*worker_before) @@ -882,7 +962,13 @@ def preservation() -> object: _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) return result except BaseException as error: + result["status"] = "failed" result["failure_type"] = type(error).__name__ + command_failure = _command_failure(error) + if command_failure is not None: + result["command_failure"] = command_failure + for diagnostic in command_failure["diagnostics"]: + logger.error("Native fixture command diagnostic: {}", diagnostic) logger.exception("Native recovery ownership fixture failed") raise finally: diff --git a/tests/test_management_archived_cutover_container.py b/tests/test_management_archived_cutover_container.py index d966db883..5aa3f8323 100644 --- a/tests/test_management_archived_cutover_container.py +++ b/tests/test_management_archived_cutover_container.py @@ -367,7 +367,7 @@ def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: b for source, target, writable in mounts: args += ["--mount", f"type=bind,src={source},dst={target}" + ("" if writable else ",readonly")] if self.original_genesis: - args += ["--env", "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python", self.image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json"] + args += ["--workdir", "/workspace", "--env", "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python", self.image, "-m", "auto_coder.management.bootstrap_supervisor", "--config", "/bootstrap/config.json"] else: args += [self.image, "--config", "/bootstrap/config.json"] if prior is not None: diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index 9dc037fb2..e652fff3d 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -10,6 +10,7 @@ import hashlib import json import os +import subprocess import sys from pathlib import Path @@ -72,6 +73,22 @@ def test_linked_credential_stops_before_host_authorization(tmp_path: Path, monke fixture.run_fixture(**inputs) +def test_command_failure_exports_bounded_diagnostics_without_inputs_or_credentials() -> None: + error = subprocess.CalledProcessError( + 1, + ["docker", "compose", "--private-command-value"], + output="Network selected Created\nError response from daemon: AppArmor profile could not be loaded\nAuthorization: Bearer secret-value\n", + stderr="stderr boundary\n" + "safe line\n" * 100, + ) + result = fixture._command_failure(error) + assert result["type"] == "CalledProcessError" and result["returncode"] == 1 + assert len(result["diagnostics"]) <= 60 + assert "private-command-value" not in json.dumps(result) and "secret-value" not in json.dumps(result) + short = fixture._command_failure(subprocess.CalledProcessError(1, ["docker", "compose"], output=b"Error response from daemon: AppArmor profile could not be loaded\n")) + assert "AppArmor profile could not be loaded" in "".join(short["diagnostics"]) + assert fixture._command_failure(ValueError("unrelated error")) is None + + def test_no_child_or_admission_oracle_rejects_any_native_store(tmp_path: Path) -> None: state, journal = tmp_path / "state", tmp_path / "journal" state.mkdir() @@ -185,6 +202,10 @@ def _result() -> dict[str, object]: "preserved_workers": True, "original_outcomes": {"C": "launched", "D": "never-created"}, "durable": {"record_sha256": {name: "a" * 64 for name in ("reserved", "intent", "consumed", "receipt")}}, + "peer_observations": { + "schema": "actual-recovery-native-peer-faults-v1", + "cases": [{"case": case, "issuer_returncode": 2, "epoch_unreserved": True, "issuer_endpoint_removed_by_original": True, "consumer_grant_accepted": False, "no_child_or_admission": True, "issuer_syscall": {"bytes_observed": 10}} for case in sorted(fixture.REQUIRED_PEER_CASES)], + }, } @@ -208,7 +229,7 @@ def test_native_result_cannot_claim_success_without_preservation(field: str) -> @pytest.mark.parametrize("scenario,phase", [("kill-after-intent", "intent"), ("kill-after-consume", "consumed"), ("lost-reply", "receipt")]) def test_interruption_result_requires_the_actual_boundary_and_no_grant(scenario: str, phase: str) -> None: result = _result() - result.update(scenario=scenario, cases=sorted(fixture.required_cases(scenario)), durable={"phase": phase, "consumer_grant_acknowledged": False}) + result.update(scenario=scenario, cases=sorted(fixture.required_cases(scenario)), durable={"phase": phase, "consumer_grant_acknowledged": False, "grant_send_completed": scenario == "lost-reply", "receiver_paused_before_send": True, "issuer_died_before_receiver_resumed": True}) fixture.validate_result(result) result["durable"]["consumer_grant_acknowledged"] = True with pytest.raises(AssertionError): @@ -223,6 +244,54 @@ def test_unknown_native_scenario_is_not_a_success_alias() -> None: fixture.required_cases("unreviewed-boundary") +def test_canonical_acceptance_contains_all_distinct_publication_and_late_faults() -> None: + assert len(set(fixture.ACCEPTANCE_SCENARIOS)) == len(fixture.ACCEPTANCE_SCENARIOS) + assert set(fixture.ACCEPTANCE_SCENARIOS) == {"grant", "lost-reply", "retarget-after-reservation", *fixture.BOUNDARY_SCENARIOS, *fixture.LATE_MUTATION_SCENARIOS} + assert set(fixture.SCENARIOS) - set(fixture.ACCEPTANCE_SCENARIOS) == {"kill-after-intent", "kill-after-consume"} + assert fixture._selected_boundary("kill-after-intent") == fixture._selected_boundary("after-intent-head") == ("intent-head.json", "after") + assert fixture._selected_boundary("kill-after-consume") == fixture._selected_boundary("after-consumed-head") == ("consumed-head.json", "after") + assert fixture._selected_boundary("lost-reply") == ("receipt-head.json", "after") + + +@pytest.mark.parametrize("field", ["grant_send_completed", "receiver_paused_before_send", "issuer_died_before_receiver_resumed"]) +def test_lost_reply_requires_actual_send_before_issuer_death(field: str) -> None: + result = _result() + result.update(scenario="lost-reply", cases=sorted(fixture.required_cases("lost-reply")), durable={"phase": "receipt", "consumer_grant_acknowledged": False, "grant_send_completed": True, "receiver_paused_before_send": True, "issuer_died_before_receiver_resumed": True}) + fixture.validate_result(result) + result["durable"][field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("case", sorted(fixture.REQUIRED_PEER_CASES)) +def test_grant_result_requires_the_actual_peer_evidence_inventory(case: str) -> None: + result = _result() + fixture.validate_result(result) + result["peer_observations"]["cases"] = [item for item in result["peer_observations"]["cases"] if item["case"] != case] + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("scenario", fixture.LATE_MUTATION_SCENARIOS) +def test_late_mutation_result_names_its_deliberate_preservation_boundary(scenario: str) -> None: + result = _result() + result.update( + scenario=scenario, + cases=sorted(fixture.required_cases(scenario)), + preservation_baseline="after-deliberate-fixture-mutation", + preserved_originals_sha256="c" * 64, + durable={"phase": "intent", "consumer_grant_acknowledged": False, "grant_send_completed": False, "mutation": {"scenario": scenario, "boundary": "after-intent-head", "preservation_baseline": "after-deliberate-fixture-mutation"}, "originals_after_injection_sha256": "c" * 64}, + ) + fixture.validate_result(result) + result["preservation_baseline"] = "before-deliberate-fixture-mutation" + with pytest.raises(AssertionError): + fixture.validate_result(result) + result["preservation_baseline"] = "after-deliberate-fixture-mutation" + result["durable"]["originals_after_injection_sha256"] = "d" * 64 + with pytest.raises(AssertionError): + fixture.validate_result(result) + + @pytest.mark.parametrize("scenario", fixture.BOUNDARY_SCENARIOS) def test_publication_result_requires_exact_kernel_observation(scenario: str) -> None: publication, when = fixture.parse_boundary(scenario) diff --git a/tests/test_management_recovery_epoch_native_peer.py b/tests/test_management_recovery_epoch_native_peer.py new file mode 100644 index 000000000..7bb90b104 --- /dev/null +++ b/tests/test_management_recovery_epoch_native_peer.py @@ -0,0 +1,172 @@ +"""Real SCM_RIGHTS ABI components, separate from native VM peer acceptance. + +The message reader inspects actual kernel-populated msghdr/iovec/control memory +from a local socketpair. No production issuer, containment or grant is asserted. +The disposable VM independently exercises the actual ptrace/process barriers. +""" + +from __future__ import annotations + +import array +import ctypes +import os +import socket +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path + +import pytest +from scripts.recovery_epoch_native_peer import ( + LATE_MUTATION_SCENARIOS, + REQUIRED_PEER_CASES, + KernelPeerMessageBarrier, + _ControlHeader, + _decode_rights, + _Iovec, + _MessageHeader, + mutate_after_intent, +) + +from auto_coder.management.recovery_epoch_peer import MAX_MESSAGE_BYTES, MAX_RESOURCES, OFFER_SCHEMA, encode_peer_message + +pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") + + +@dataclass +class _OwnMemory: + pid: int + returncode: int | None = None + + def wait(self, timeout: float | None = None) -> int: + raise AssertionError("this parser component cannot own a tracee") + + +@contextmanager +def _kernel_message(tmp_path: Path, count: int): + """Populate the actual Linux ABI by invoking libc recvmsg, without ptrace.""" + libc = ctypes.CDLL(None, use_errno=True) + libc.recvmsg.argtypes = [ctypes.c_int, ctypes.POINTER(_MessageHeader), ctypes.c_int] + libc.recvmsg.restype = ctypes.c_ssize_t + packet = encode_peer_message({"schema": OFFER_SCHEMA, "request_id": "request", "authorization_id": "authorization", "target_id": "target", "resources": []}) + data = ctypes.create_string_buffer(MAX_MESSAGE_BYTES + 1) + control = ctypes.create_string_buffer(socket.CMSG_SPACE(MAX_RESOURCES * array.array("i").itemsize)) + vector = _Iovec(ctypes.addressof(data), len(data)) + header = _MessageHeader(None, 0, ctypes.addressof(vector), 1, ctypes.addressof(control), len(control), 0) + receiver_fds: tuple[int, ...] = () + with ExitStack() as held: + sender, receiver = socket.socketpair() + held.enter_context(sender) + held.enter_context(receiver) + sources = tuple(held.enter_context((tmp_path / str(index)).open("w+b")).fileno() for index in range(count)) + ancillary = [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", sources))] if count else [] + assert sender.sendmsg([packet], ancillary) == len(packet) + result = libc.recvmsg(receiver.fileno(), ctypes.byref(header), socket.MSG_CMSG_CLOEXEC) + assert result == len(packet), os.strerror(ctypes.get_errno()) + if count: + # Capture only the actual expected received integers for cleanup, + # independently of the parser under test's handling of cmsghdr. + values = array.array("i") + values.frombytes(control.raw[ctypes.sizeof(_ControlHeader) : ctypes.sizeof(_ControlHeader) + count * values.itemsize]) + receiver_fds = tuple(values) + reader = object.__new__(KernelPeerMessageBarrier) + reader.issuer = _OwnMemory(os.getpid()) + reader.binding = {"request_id": "request", "authorization_id": "authorization", "target_id": "target"} + reader.schema = OFFER_SCHEMA + try: + yield reader, header, result, packet, sources, receiver_fds + finally: + for descriptor in receiver_fds: + os.close(descriptor) + + +@pytest.mark.parametrize("count", [0, 1, 2, MAX_RESOURCES]) +def test_real_recvmsg_memory_retains_exact_bytes_and_received_kernel_objects(tmp_path: Path, count: int) -> None: + with _kernel_message(tmp_path, count) as (reader, header, size, packet, originals, received): + encoded, descriptors = reader._message(ctypes.addressof(header), received_length=size) + assert encoded == packet and descriptors == received + assert reader._selected(encoded) + for original, actual in zip(originals, received): + assert original != actual + assert (os.fstat(original).st_dev, os.fstat(original).st_ino) == (os.fstat(actual).st_dev, os.fstat(actual).st_ino) + assert os.get_inheritable(actual) is False + + +@pytest.mark.parametrize("field,value", [("vector_count", 0), ("vector_count", 5), ("control_length", 4096), ("flags", socket.MSG_CTRUNC), ("flags", socket.MSG_TRUNC)]) +def test_actual_message_reader_refuses_incomplete_or_unbounded_abi_observations(tmp_path: Path, field: str, value: int) -> None: + with _kernel_message(tmp_path, 1) as (reader, header, size, _packet, _originals, _received): + setattr(header, field, value) + with pytest.raises(AssertionError, match="vectors or ancillary"): + reader._message(ctypes.addressof(header), received_length=size) + + +def test_actual_message_reader_refuses_a_return_larger_than_receiver_capacity(tmp_path: Path) -> None: + with _kernel_message(tmp_path, 1) as (reader, header, _size, _packet, _originals, _received): + with pytest.raises(AssertionError, match="iovec capacity"): + reader._message(ctypes.addressof(header), received_length=MAX_MESSAGE_BYTES + 2) + + +@pytest.mark.parametrize("scenario", ["", "late", "late-arbitrary-file", "late-../epoch", "after-intent-head"]) +def test_late_mutation_selector_refuses_unreviewed_actions_before_access(scenario: str) -> None: + with pytest.raises(ValueError, match="reviewed late mutation"): + with mutate_after_intent(None, scenario, os.getpid()): + pytest.fail("an unreviewed mutation was admitted") + + +def test_native_case_sets_are_explicit_and_distinct() -> None: + assert len(REQUIRED_PEER_CASES) == 4 + assert len(LATE_MUTATION_SCENARIOS) == 10 and len(set(LATE_MUTATION_SCENARIOS)) == 10 + assert not REQUIRED_PEER_CASES.intersection(LATE_MUTATION_SCENARIOS) + + +@pytest.mark.parametrize("malformation", ["header", "length", "type", "integer", "duplicate", "negative", "too-many"]) +def test_native_control_parser_refuses_nonexact_descriptor_records(malformation: str) -> None: + descriptor_data = array.array("i", [3]).tobytes() + header = _ControlHeader(ctypes.sizeof(_ControlHeader) + len(descriptor_data), socket.SOL_SOCKET, socket.SCM_RIGHTS) + if malformation == "header": + encoded = b"\x00" + else: + if malformation == "length": + header.length += 32 + elif malformation == "type": + header.kind = socket.SCM_CREDENTIALS + elif malformation == "integer": + descriptor_data += b"\x00" + header.length += 1 + elif malformation in {"duplicate", "negative", "too-many"}: + descriptor_data = array.array("i", [3, 3] if malformation == "duplicate" else [-1] if malformation == "negative" else range(MAX_RESOURCES + 1)).tobytes() + header.length = ctypes.sizeof(_ControlHeader) + len(descriptor_data) + encoded = bytes(header) + descriptor_data + with pytest.raises(AssertionError, match="native"): + _decode_rights(encoded) + + +def test_real_shared_ofd_reacquisition_changes_kernel_acquirer_visible_to_original_process(tmp_path: Path) -> None: + """Check the kernel premise of the VM fault, without substituting pidfd_getfd.""" + import fcntl + import select + import subprocess + import sys + + from auto_coder.management.recovery_epoch_peer import inspect_kernel_lock + + script = "import fcntl,sys; fd=int(sys.argv[1]); fcntl.flock(fd, fcntl.LOCK_SH); print('original-acquired',flush=True); sys.stdin.read(1)" + with (tmp_path / "original-owner.lock").open("w+b") as original: + os.fchmod(original.fileno(), 0o600) + info = os.fstat(original.fileno()) + process = subprocess.Popen([sys.executable, "-I", "-B", "-c", script, str(original.fileno())], pass_fds=(original.fileno(),), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert select.select((process.stdout,), (), (), 10)[0] + assert process.stdout.readline() == "original-acquired\n" + before = inspect_kernel_lock(process.pid, original.fileno(), info.st_dev, info.st_ino) + assert before is not None and (before.mode, before.pid) == ("READ", process.pid) + fcntl.flock(original.fileno(), fcntl.LOCK_UN) + fcntl.flock(original.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + after = inspect_kernel_lock(process.pid, original.fileno(), info.st_dev, info.st_ino) + assert after is not None and (after.mode, after.pid) == ("READ", os.getpid()) + assert after != before and after == inspect_kernel_lock(os.getpid(), original.fileno(), info.st_dev, info.st_ino) + process.communicate("x", timeout=10) + assert process.returncode == 0 + finally: + if process.poll() is None: + process.kill() + process.communicate(timeout=10) From 69b6eede4e07fa56d4daa834fb16cd7bb077a378 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 01:35:20 +0900 Subject: [PATCH 06/24] test(management): close native recovery oracle gaps --- .../management-recovery-epoch.md | 29 ++ scripts/recovery_epoch_fault_tracer.py | 5 +- scripts/recovery_epoch_native_negatives.py | 328 +++++++++++++++++- scripts/recovery_epoch_native_peer.py | 163 ++++++++- scripts/recovery_vm/README.md | 4 +- scripts/recovery_vm/guest.sh | 8 +- .../test_management_recovery_epoch_host.py | 68 +++- tests/test_management_recovery_epoch_host.py | 110 +++++- ...agement_recovery_epoch_native_negatives.py | 109 +++++- ...t_management_recovery_epoch_native_peer.py | 181 +++++++++- 10 files changed, 971 insertions(+), 34 deletions(-) diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md index e2ca81c2c..e185bd7a3 100644 --- a/docs/client-features/management-recovery-epoch.md +++ b/docs/client-features/management-recovery-epoch.md @@ -198,3 +198,32 @@ observable contention, missing/mutated/forged prerequisites, all-controller/task listener refusals and interruption/host-death boundaries. Passing the component selection alone does not establish those native obligations or applicability to a current incident with missing originals. + +### Maintained native acceptance + +`scripts/test_management_recovery_epoch_host.py` owns the actual host-entrypoint +driver and each scenario's required outcomes. The +`Management Recovery Epoch Acceptance` workflow derives its complete matrix from +that driver's `ACCEPTANCE_SCENARIOS`; its final job requires every scenario to +succeed. A missing case, failed VM, unavailable native kernel operation or failed +preservation check cannot become a passed result. The VM runner and reviewed +reproduction procedure are documented in `scripts/recovery_vm/README.md`. + +| Contract | Actual fixture action and independent outcome | +| --- | --- | +| AS-B01 | Run genuine genesis, successful successor, never-created failed branch and unresolved successor producers. Verify full original lineage, reject forged/cross-deployment/stale reports, remove an original journal, and mutate the retained failed branch's exact terminal evidence after intent. Refusal must preserve the original uncertainty and stores. | +| AS-B02 | Install the ordinary service/timer/watcher/alternate-root fixture routes, bind them in approved enrollment and verify their actual inhibition. Exercise unknown/omitted/startable controllers, retained namespace listeners, an escaped nonleader task and independently measured exit. Repeat the real containment verifier in a separate process and preserve unrelated worker claims and effect counters. | +| AS-B03 | Use the real host and Docker bootstrap processes, kernel peer credentials/pidfds and independent SH/target EX acquisition. Refuse busy/replaced/missing fences, a foreign acquiring process, queued rights without acknowledgment, received rights before acknowledgment, and a changed target lifetime. Later successful issuance must retain the complete observed custody. | +| AS-B04 | Reject actual CLI/environment root and inventory overrides. Stop the real issuer at its fsynced reservation head while a contender runs, then test conflicting target/preparation binding. In separate fresh VMs, SIGKILL the original process before and after every record/head publication and inspect the exact retained prefix and refused replay. | +| AS-B05 | After the original intent head, change exact configuration, READY, installed source, original evidence, watcher/controller state, fence acquisition or target lifetime. Require refusal before consumption. Separately stop the receiver, require the real grant send to succeed, prove issuer death through its pidfd, then resume the receiver and require unavailable current custody with no replay. | + +Publication and protocol fault helpers only observe real syscall entry/exit and +kernel descriptors. They do not replace system-call results, inject verifier +verdicts or alter production code. Deliberate late fixture mutations remain +present through refusal and replay; their exported preservation baseline is +explicitly the state after that injection. The successful path measures original +OAuth/history/journal bytes and identities, unrelated beta/release/agent-table +worker ownership and effect requests, and the absence of any replacement child +or application admission. The current PR's custody endpoint and a separately +qualified installed READY are exercised independently; these checks do not claim +that the separate #2825 management-child launch/readiness contract is complete. diff --git a/scripts/recovery_epoch_fault_tracer.py b/scripts/recovery_epoch_fault_tracer.py index f843cb17d..10ddaa553 100644 --- a/scripts/recovery_epoch_fault_tracer.py +++ b/scripts/recovery_epoch_fault_tracer.py @@ -37,6 +37,7 @@ _AUDIT_ARCH_X86_64 = 0xC000003E _SYS_OPENAT = 257 _SYS_FSYNC = 74 +_BARRIER_ACTION_TIMEOUT = 1800 class _Entry(ctypes.Structure): @@ -234,7 +235,7 @@ def _run(self) -> None: if not os.WIFSTOPPED(status): raise AssertionError("actual issuer exited before the initial trace barrier") self._ready.set() - if not self._resume.wait(180): + if not self._resume.wait(_BARRIER_ACTION_TIMEOUT): raise TimeoutError("the actual target probe did not release the initial barrier") self._ptrace(_PTRACE_SYSCALL) while True: @@ -245,7 +246,7 @@ def _run(self) -> None: if delivered == signal.SIGTRAP | 0x80: if self._observe_call(): self._record() - if not self._action.wait(180): + if not self._action.wait(_BARRIER_ACTION_TIMEOUT): raise TimeoutError("the observed publication was not explicitly released or killed") if self._decision == "release": self._ptrace(_PTRACE_DETACH) diff --git a/scripts/recovery_epoch_native_negatives.py b/scripts/recovery_epoch_native_negatives.py index ef7d26687..3327a9aef 100644 --- a/scripts/recovery_epoch_native_negatives.py +++ b/scripts/recovery_epoch_native_negatives.py @@ -22,12 +22,12 @@ import sys import time import uuid -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from dataclasses import asdict, dataclass from pathlib import Path from typing import Callable, ContextManager, Iterator -from scripts.test_management_containment_closed_host import MARKER, read_authorization +from scripts.test_management_containment_closed_host import _NONLEADER, MARKER, read_authorization from scripts.test_management_containment_enrollment_host import _docker, _systemctl, _write_private from auto_coder.logger_config import get_logger @@ -45,17 +45,27 @@ logger = get_logger(__name__) _FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") _UNRESERVED_ENTRIES = frozenset({"epoch.lock", "inventory.json"}) +_FACTORY_ENVIRONMENT_OVERRIDES = ( + ("factory-epoch-root-environment-refused", "AUTO_CODER_RECOVERY_EPOCH_ROOT"), + ("factory-inventory-environment-refused", "AUTO_CODER_RECOVERY_EPOCH_INVENTORY"), + ("factory-authorization-environment-refused", "AUTO_CODER_RECOVERY_AUTHORIZATION_ROOT"), + ("factory-containment-environment-refused", "AUTO_CODER_RECOVERY_CONTAINMENT_ROOT"), +) +_FACTORY_ARGUMENT_OVERRIDES = (("factory-root-argument-refused", "--root"), ("factory-inventory-argument-refused", "--inventory")) REQUIRED_NEGATIVE_CASES = frozenset( { "busy-original-owner-refused", "busy-controller-fence-refused", + "missing-original-owner-refused", "replaced-original-owner-refused", "replaced-epoch-lock-refused", "unknown-docker-management-route-refused", "omitted-installed-controller-refused", "unmasked-installed-restarter-refused", "retained-old-network-task-and-listener-refused", + "escaped-old-network-nonleader-refused", "old-published-listener-refused", + *(label for label, _ in _FACTORY_ENVIRONMENT_OVERRIDES + _FACTORY_ARGUMENT_OVERRIDES), } ) REQUIRED_REPORT_CASES = frozenset( @@ -64,6 +74,8 @@ "forged-native-containment-refused", "cross-deployment-containment-refused", "stale-native-containment-refused", + "escaped-nonleader-current-containment-refused", + "measured-exited-nonleader-current-containment-reverified", } ) @@ -166,6 +178,27 @@ def _replacement(epoch_root: Path, path: Path, backup: Path, contents: bytes) -> raise AssertionError("fixture cleanup did not restore the exact original object") +@contextmanager +def _missing_original(epoch_root: Path, path: Path, backup: Path) -> Iterator[None]: + """Remove only the selected fixture path while retaining its original inode.""" + assert_unreserved_epoch(epoch_root) + original = _projection(path) + if path != path.resolve(strict=True) or backup.exists() or backup.is_symlink() or backup.parent.stat().st_dev != path.parent.stat().st_dev: + raise AssertionError("fixture absence requires a unique same-filesystem backup") + path.rename(backup) + try: + yield + finally: + assert_unreserved_epoch(epoch_root) + if _projection(backup) != original: + raise AssertionError("the absent original changed during the negative fixture") + if path.exists() or path.is_symlink(): + raise AssertionError("the missing original was recreated; refusing fixture cleanup") + backup.rename(path) + if _projection(path) != original: + raise AssertionError("fixture absence cleanup did not restore the exact original") + + @contextmanager def _busy_fence(selected: RecoveryLock) -> Iterator[None]: descriptor = os.open(selected.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) @@ -265,6 +298,200 @@ def _retained_namespace_listener(original: RecoveryOriginal) -> Iterator[None]: os.close(descriptor) +@dataclass(frozen=True) +class NativeTaskObservation: + pid: int + tid: int + start_token: str + state: str + network_namespace: str | None + cgroups: tuple[str, ...] + + +@dataclass(frozen=True) +class EscapedNonleaderObservation: + generation_id: str + original_namespace: str + original_cgroup: str + leader: NativeTaskObservation + nonleader: NativeTaskObservation + + +@dataclass(frozen=True) +class ExitedNonleaderObservation: + leader: NativeTaskObservation + nonleader_tid: int + nonleader_absent: bool + remaining_tids: tuple[int, ...] + wait_code: int + wait_status: int + pidfd_readable: bool + + +def _task_observation(pid: int, tid: int) -> NativeTaskObservation: + """Read one actual kernel task twice; missing namespace alone is not exit.""" + if type(pid) is not int or type(tid) is not int or min(pid, tid) <= 1: + raise AssertionError("native task observation requires exact positive identities") + task = Path(f"/proc/{pid}/task/{tid}") + + def identity() -> tuple[str, str]: + value = (task / "stat").read_text(encoding="ascii") + prefix, separator, tail = value.rpartition(")") + fields = tail.split() + if len(value) > 4096 or not separator or prefix.partition(" (")[0] != str(tid) or len(fields) < 20 or not fields[19].isdecimal() or int(fields[19]) <= 0: + raise AssertionError("the actual task lifetime is incomplete") + return fields[0], fields[19] + + before = identity() + try: + namespace = os.readlink(task / "ns/net") + except FileNotFoundError: + namespace = None + groups = tuple((task / "cgroup").read_text(encoding="ascii").splitlines()) + after = identity() + if before[1] != after[1] or (before[0] == "Z") != (after[0] == "Z") or len(groups) != 1 or not groups[0].startswith("0::/") or namespace is not None and re.fullmatch(r"net:\[[0-9]+\]", namespace) is None: + raise AssertionError("the actual task changed or left the measured kernel profile") + return NativeTaskObservation(pid, tid, after[1], after[0], namespace, groups) + + +def _escaped_task_observation(original: RecoveryOriginal, pid: int, tid: int) -> EscapedNonleaderObservation: + pin = original.namespace_pin + if pin is None or original.prior_evidence is None or tid == pid: + raise AssertionError("a real nonleader and original namespace are required") + prior, _, _ = _evidence(Path(original.prior_evidence.path)) + leader, nonleader = _task_observation(pid, pid), _task_observation(pid, tid) + if leader.network_namespace != os.readlink("/proc/self/ns/net") or leader.network_namespace == pin.namespace or nonleader.network_namespace != pin.namespace: + raise AssertionError("the escaped nonleader lacks the independently observed namespace split") + for task in (leader, nonleader): + group = task.cgroups[0][3:] + if task.state in {"Z", "X", "x"} or group == prior.cgroup_path or group.startswith(prior.cgroup_path + "/"): + raise AssertionError("the live escaped task is not outside the original cgroup") + return EscapedNonleaderObservation(original.generation_id, pin.namespace, prior.cgroup_path, leader, nonleader) + + +def _same_live_task(left: NativeTaskObservation, right: NativeTaskObservation) -> bool: + return (left.pid, left.tid, left.start_token, left.network_namespace, left.cgroups) == (right.pid, right.tid, right.start_token, right.network_namespace, right.cgroups) and right.state not in {"Z", "X", "x"} + + +def _wait_unreaped_exit(pid: int, descriptor: int, start_token: str, nonleader_tid: int) -> ExitedNonleaderObservation: + """Keep the exact zombie available for a separate production verifier.""" + poller = select.poll() + poller.register(descriptor, select.POLLIN) + if not poller.poll(15000): + raise AssertionError("the exact nonleader process did not exit at its pidfd") + waited = os.waitid(os.P_PIDFD, descriptor, os.WEXITED | os.WNOWAIT | os.WNOHANG) + if waited is None or waited.si_pid != pid or waited.si_code not in {os.CLD_EXITED, os.CLD_KILLED, os.CLD_DUMPED}: + raise AssertionError("the original child has no independently observed unreaped exit") + leader = _task_observation(pid, pid) + tasks = tuple(sorted(int(path.name) for path in Path(f"/proc/{pid}/task").iterdir())) + absent = not Path(f"/proc/{pid}/task/{nonleader_tid}").exists() + if nonleader_tid == pid or leader.start_token != start_token or leader.state != "Z" or leader.network_namespace is not None or not absent or tasks != (pid,): + raise AssertionError("the original task is not a measured namespace-free zombie with its nonleader gone") + if _task_observation(pid, pid) != leader or not poller.poll(0): + raise AssertionError("the exact exited task changed during independent observation") + return ExitedNonleaderObservation(leader, nonleader_tid, absent, tasks, waited.si_code, waited.si_status, True) + + +@dataclass +class NativeNonleaderProcess: + process: subprocess.Popen[bytes] + descriptor: int + original: RecoveryOriginal + before: EscapedNonleaderObservation + exited: ExitedNonleaderObservation | None = None + + def observe_live(self) -> EscapedNonleaderObservation: + poller = select.poll() + poller.register(self.descriptor, select.POLLIN) + if self.exited is not None or poller.poll(0): + raise AssertionError("the original escaped nonleader ended before refusal") + actual = _escaped_task_observation(self.original, self.process.pid, self.before.nonleader.tid) + if not _same_live_task(self.before.leader, actual.leader) or not _same_live_task(self.before.nonleader, actual.nonleader): + raise AssertionError("the exact escaped nonleader lifetime or custody changed") + return actual + + def exit_unreaped(self) -> ExitedNonleaderObservation: + self.observe_live() + signal.pidfd_send_signal(self.descriptor, signal.SIGTERM) + self.exited = _wait_unreaped_exit(self.process.pid, self.descriptor, self.before.leader.start_token, self.before.nonleader.tid) + return self.exited + + def observe_exited(self) -> ExitedNonleaderObservation: + if self.exited is None: + raise AssertionError("no measured task exit precedes the positive control") + actual = _wait_unreaped_exit(self.process.pid, self.descriptor, self.before.leader.start_token, self.before.nonleader.tid) + if actual != self.exited: + raise AssertionError("the unreaped original lifetime changed during the positive control") + return actual + + +@contextmanager +def _escaped_namespace_nonleader(original: RecoveryOriginal) -> Iterator[NativeNonleaderProcess]: + """Reuse the real closed-host thread producer with exact original pin custody.""" + pin = original.namespace_pin + if pin is None or original.prior_evidence is None or _holder_start_token(pin.holder_pid) != pin.holder_start_token: + raise AssertionError("the original retained namespace holder is unavailable") + namespace = os.open(f"/proc/{pin.holder_pid}/fd/{pin.holder_fd}", os.O_RDONLY | os.O_CLOEXEC) + process: subprocess.Popen[bytes] | None = None + lifetime = -1 + try: + info = os.fstat(namespace) + if (info.st_dev, info.st_ino) != (pin.device, pin.inode) or os.readlink(f"/proc/self/fd/{namespace}") != pin.namespace: + raise AssertionError("the nonleader fixture did not retain the exact original namespace") + process = subprocess.Popen((sys.executable, "-I", "-B", "-c", _NONLEADER, str(namespace)), pass_fds=(namespace,), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + lifetime = os.pidfd_open(process.pid, 0) + assert process.stdout is not None + message = b"" + deadline = time.monotonic() + 15 + while b"\n" not in message and len(message) <= 256: + left = deadline - time.monotonic() + if left <= 0 or not select.select((process.stdout,), (), (), left)[0]: + raise AssertionError("the real escaped nonleader did not reach its readiness barrier") + chunk = os.read(process.stdout.fileno(), 257 - len(message)) + if not chunk: + raise AssertionError("the real escaped nonleader exited before readiness") + message += chunk + ready = json.loads(message) + if not message.endswith(b"\n") or len(message) > 256 or not isinstance(ready, dict) or set(ready) != {"pid", "tid"} or ready["pid"] != process.pid or type(ready["tid"]) is not int or ready["tid"] == process.pid: + raise AssertionError("the real escaped nonleader readiness identity differs") + held = NativeNonleaderProcess(process, lifetime, original, _escaped_task_observation(original, process.pid, ready["tid"])) + held.observe_live() + yield held + if held.exited is None: + held.observe_live() + else: + held.observe_exited() + finally: + if process is not None: + if lifetime >= 0: + poller = select.poll() + poller.register(lifetime, select.POLLIN) + if not poller.poll(0): + signal.pidfd_send_signal(lifetime, signal.SIGTERM) + elif process.poll() is None: + process.terminate() + try: + process.communicate(timeout=15) + except subprocess.TimeoutExpired: + if lifetime >= 0: + signal.pidfd_send_signal(lifetime, signal.SIGKILL) + else: + process.kill() + process.communicate(timeout=15) + if lifetime >= 0: + os.close(lifetime) + os.close(namespace) + + +@contextmanager +def _record_nonleader_refusal(context: NativeNegativeContext, original: RecoveryOriginal) -> Iterator[None]: + with _escaped_namespace_nonleader(original) as held: + before = held.observe_live() + yield + after = held.observe_live() + _write_private(context.fixture_root / "native-nonleader-issuer-observation.json", canonical({"case": "escaped-old-network-nonleader-refused", "before": asdict(before), "after": asdict(after), "epoch_sha256": _epoch_snapshot(context.epoch_root)})) + + @contextmanager def _published_listener(original: RecoveryOriginal, target_port: int) -> Iterator[None]: if original.prior_evidence is None: @@ -375,6 +602,79 @@ def _run_case(context: NativeNegativeContext, label: str, injection: Callable[[] logger.info("Native prepublication refusal and preservation verified: {}", label) +@dataclass(frozen=True) +class FactoryOverrideObservation: + case: str + argument: str + environment_variable: str + selected_path: str + returncode: int + refusal: str + alternate_inputs_sha256: str + epoch_sha256: str + + +def _alternate_input_snapshot(root: Path) -> str: + """Bind the real alternate directory, including new or removed entries.""" + if root != root.resolve(strict=True): + raise AssertionError("alternate fixture inputs are no longer canonical") + paths = (root, *sorted(root.rglob("*"))) + if len(paths) > 32: + raise AssertionError("alternate fixture input inventory exceeds its bound") + entries = [] + for path in paths: + info = path.lstat() + if stat.S_ISREG(info.st_mode): + contents = hashlib.sha256(path.read_bytes()).hexdigest() + elif stat.S_ISDIR(info.st_mode): + contents = "directory" + else: + raise AssertionError("alternate fixture input has an unsupported object") + entries.append((str(path.relative_to(root)), info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns, contents)) + return digest(entries) + + +def _run_factory_overrides(context: NativeNegativeContext, preserved_snapshot: Callable[[], object]) -> list[str]: + """Reject real CLI selectors before either fixed or alternate authority moves.""" + if any(name in os.environ for _, name in _FACTORY_ENVIRONMENT_OVERRIDES): + raise AssertionError("the native fixture environment already contains an authority override") + directory = context.fixture_root / "native-factory-overrides" + directory.mkdir(mode=0o700) + alternate = directory / "alternate-authority" + alternate.mkdir(mode=0o700) + for name in sorted(_UNRESERVED_ENTRIES): + _write_private(alternate / name, (context.epoch_root / name).read_bytes()) + observed: list[FactoryOverrideObservation] = [] + selectors = tuple((label, argument, "") for label, argument in _FACTORY_ARGUMENT_OVERRIDES) + tuple((label, "", variable) for label, variable in _FACTORY_ENVIRONMENT_OVERRIDES) + for label, argument, variable in selectors: + selected = alternate / "inventory.json" if argument == "--inventory" or variable == "AUTO_CODER_RECOVERY_EPOCH_INVENTORY" else alternate + + def refuse(actual_label: str) -> None: + before = _alternate_input_snapshot(alternate) + environment = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + command = [sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"] + if argument: + command.extend((argument, str(selected))) + else: + environment[variable] = str(selected) + refusal = "unrecognized arguments: " + argument if argument else "request/environment recovery inventory overrides are unsupported" + try: + completed = subprocess.run(command, env=environment, capture_output=True, text=True, timeout=30) + output = completed.stdout + completed.stderr + _write_private(directory / (actual_label + ".log"), output.encode()) + if completed.returncode != 2 or refusal not in output or "Fixed recovery issuer endpoint is listening" in output or "One consumed recovery epoch grant send completed" in output: + raise AssertionError("the actual production issuer accepted or misclassified a factory override: " + actual_label) + finally: + assert_unreserved_epoch(context.epoch_root) + if _alternate_input_snapshot(alternate) != before: + raise AssertionError("the actual production issuer changed alternate authority inputs") + observed.append(FactoryOverrideObservation(actual_label, argument, variable, str(selected), completed.returncode, refusal, before, _epoch_snapshot(context.epoch_root))) + + _run_case(context, label, nullcontext, refuse, preserved_snapshot) + _write_private(context.fixture_root / "native-factory-override-observations.json", canonical({"schema": "actual-recovery-factory-overrides-v1", "cases": [asdict(item) for item in observed]})) + return [item.case for item in observed] + + def run_native_negatives(context: NativeNegativeContext, run_refusal: Callable[[str], None], preserved_snapshot: Callable[[], object]) -> list[str]: """Exercise actual issuer refusals before its first irreversible publication.""" require_native_context(context) @@ -389,18 +689,21 @@ def run_native_negatives(context: NativeNegativeContext, run_refusal: Callable[[ injections: tuple[tuple[str, Callable[[], ContextManager[None]]], ...] = ( ("busy-original-owner-refused", lambda: _busy_fence(selected_owner)), ("busy-controller-fence-refused", lambda: _busy_fence(context.enrollment.watchers[0].lock)), + ("missing-original-owner-refused", lambda: _missing_original(context.epoch_root, owner, backups / "absent-owner.original")), ("replaced-original-owner-refused", lambda: _replacement(context.epoch_root, owner, backups / "owner.original", owner.read_bytes())), ("replaced-epoch-lock-refused", lambda: _replacement(context.epoch_root, epoch, backups / "epoch.original", epoch.read_bytes())), ("unknown-docker-management-route-refused", lambda: _extra_docker(context.enrollment)), ("omitted-installed-controller-refused", lambda: _omitted_controller(context, backups)), ("unmasked-installed-restarter-refused", lambda: _unmasked_restarter(context)), ("retained-old-network-task-and-listener-refused", lambda: _retained_namespace_listener(original)), + ("escaped-old-network-nonleader-refused", lambda: _record_nonleader_refusal(context, original)), ("old-published-listener-refused", lambda: _published_listener(original, context.enrollment.target.published_port)), ) cases = [] for label, injection in injections: _run_case(context, label, injection, run_refusal, preserved_snapshot) cases.append(label) + cases.extend(_run_factory_overrides(context, preserved_snapshot)) if set(cases) != REQUIRED_NEGATIVE_CASES: raise AssertionError("the native refusal matrix is incomplete") return cases @@ -500,9 +803,28 @@ def probe(label: str, *, previous: Path | None = None, refusal: str | None = Non with _retained_namespace_listener(original): probe("stale-native-containment-refused", previous=previous, refusal="prior network namespace still has a task") cases.append("stale-native-containment-refused") - probe("restored-current-verifier", previous=previous) + restored = probe("restored-current-verifier", previous=previous) + restored_path = directory / "pre-nonleader-observation.json" + _write_private(restored_path, canonical(restored)) + issuer_observation = json.loads((context.fixture_root / "native-nonleader-issuer-observation.json").read_bytes()) + if not isinstance(issuer_observation, dict) or issuer_observation.get("case") != "escaped-old-network-nonleader-refused": + raise AssertionError("the actual full-issuer nonleader observation is absent") + measurements = [issuer_observation] + with _escaped_namespace_nonleader(original) as held: + before = held.observe_live() + refused = probe("escaped-nonleader-current-containment-refused", previous=restored_path, refusal="prior network namespace still has a task") + after = held.observe_live() + measurements.append({"case": "escaped-nonleader-current-containment-refused", "before": asdict(before), "after": asdict(after), "status": refused["status"]}) + cases.append("escaped-nonleader-current-containment-refused") + exited = held.exit_unreaped() + held.observe_exited() + verified = probe("measured-exited-nonleader-current-containment-reverified", previous=restored_path) + retained_exit = held.observe_exited() + measurements.append({"case": "measured-exited-nonleader-current-containment-reverified", "before_exit": asdict(after), "after_exit": asdict(exited), "after_verification": asdict(retained_exit), "status": verified["status"]}) + cases.append("measured-exited-nonleader-current-containment-reverified") if set(cases) != REQUIRED_REPORT_CASES: raise AssertionError("the actual report-verifier case matrix is incomplete") + _write_private(context.fixture_root / "native-task-observations.json", canonical({"schema": "actual-recovery-task-observations-v1", "cases": measurements})) return cases finally: check_barrier() diff --git a/scripts/recovery_epoch_native_peer.py b/scripts/recovery_epoch_native_peer.py index c874f38a9..24dca67f4 100644 --- a/scripts/recovery_epoch_native_peer.py +++ b/scripts/recovery_epoch_native_peer.py @@ -14,6 +14,7 @@ import hashlib import json import os +import re import select import signal import socket @@ -33,7 +34,8 @@ from auto_coder.logger_config import get_logger from auto_coder.management.bootstrap_supervisor import BootstrapConfig from auto_coder.management.containment_route_manifest import _systemd_unit -from auto_coder.management.original_attempts import canonical +from auto_coder.management.original_attempt_evidence import OriginalLineageEvidence, derive_lineage, observe, verify_lineage +from auto_coder.management.original_attempts import canonical, read_original from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, PEER_SOCKET, EpochAuthorization from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA, ConsumerConfiguration, parse_consumer_configuration from auto_coder.management.recovery_epoch_peer import CHALLENGE_SCHEMA, MAX_MESSAGE_BYTES, MAX_RESOURCES, OFFER_SCHEMA, RECEIPT_SCHEMA, inspect_kernel_lock @@ -51,6 +53,7 @@ "late-original-fence", "late-target-fence", "late-controller-fence", + "late-target-loss", ) _SYS_SENDTO = 44 _SYS_SENDMSG = 46 @@ -140,6 +143,63 @@ def close(self) -> None: self.pidfd = -1 +def _container_ids(encoded: str) -> tuple[str, ...]: + """Require complete real Docker identities before interpreting absence.""" + values = tuple(encoded.splitlines()) + if len(values) > 4096 or len(values) != len(set(values)) or any(re.fullmatch(r"[0-9a-f]{64}", value) is None for value in values): + raise AssertionError("the actual Docker inventory has incomplete or conflicting identities") + return tuple(sorted(values)) + + +def _verify_target_loss(pidfd: int, identifier: str, before: tuple[str, ...], after: tuple[str, ...]) -> None: + """Require both kernel death and removal of exactly the selected object.""" + if before.count(identifier) != 1 or after != tuple(value for value in before if value != identifier): + raise AssertionError("the exact target was not removed without changing other Docker objects") + poll = select.poll() + poll.register(pidfd, select.POLLIN) + if not poll.poll(0): + raise AssertionError("the original target pidfd has not independently reported death") + + +@contextmanager +def _remove_target(context: NativeNegativeContext) -> Iterator[dict[str, object]]: + """Remove only the pinned disposable target and retain that loss on exit.""" + target = DockerTraceProcess(context) + try: + identifier = context.enrollment.target.management_id + before = _container_ids(_docker("ps", "--all", "--quiet", "--no-trunc")) + if identifier not in before: + raise AssertionError("the exact target is missing before its deliberate removal") + target.check() + # _docker checks the actual command return code. Neither a failed + # removal nor Docker's stopped flag supplies kernel lifetime evidence. + removed = _docker("rm", "--force", identifier, timeout=60) + if removed != identifier: + raise AssertionError("the actual Docker removal did not identify the selected target") + _wait(target.dead, "the original target pidfd reports actual death") + after = _container_ids(_docker("ps", "--all", "--quiet", "--no-trunc")) + _verify_target_loss(target.pidfd, identifier, before, after) + observation = { + "target_id": identifier, + "pid": target.pid, + "start_token": target.start_token, + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "docker_started_at": target.docker_started_at, + "docker_rm_returncode": 0, + "pidfd_death_observed": True, + "exact_docker_id_absent": True, + "other_docker_ids_preserved": True, + "container_ids_before_sha256": hashlib.sha256(canonical(before)).hexdigest(), + "container_ids_after_sha256": hashlib.sha256(canonical(after)).hexdigest(), + } + yield observation + # The driver must retain the injected loss through the original + # refusal and fixed-root replay; it must never recreate this target. + _verify_target_loss(target.pidfd, identifier, before, _container_ids(_docker("ps", "--all", "--quiet", "--no-trunc"))) + finally: + target.close() + + def _consumer_configuration(context: NativeNegativeContext) -> ConsumerConfiguration: selected = [item for item in context.enrollment.target.mounts if item.destination == "/bootstrap/recovery-epoch.json"] if len(selected) != 1 or selected[0].writable: @@ -241,6 +301,59 @@ def _require_publication_prefix(context: NativeNegativeContext, issuer_pid: int, raise AssertionError("the durable original prefix belongs to another issuer lifetime") +def _failed_terminal(lineage: OriginalLineageEvidence) -> tuple[Path, dict[str, object]]: + """Select a failed original only from the complete, independently read set.""" + branches = [generation for generation in lineage.generations if generation.outcome == "never-created"] + if lineage.unavailable or len(branches) != 1: + raise AssertionError("the actual fixture has no unique complete never-created failed branch") + branch = branches[0] + matches = [] + for original in lineage.originals: + path = Path(original.path) + if path.parent != Path(lineage.binding.root) or path.suffix != ".json": + continue + encoded, file_id = read_original(path) + if file_id != (original.device, original.inode) or hashlib.sha256(encoded).hexdigest() != original.sha256: + raise AssertionError("the selected failed-branch original changed after independent derivation") + value = json.loads(encoded) + if isinstance(value, dict) and value.get("kind") == "never-created" and value.get("attempt") == branch.request_id: + matches.append((path, original)) + if len(matches) != 1: + raise AssertionError("the actual failed branch has no unique matched original terminal record") + path, original = matches[0] + return path, {"instance_id": branch.instance_id, "request_id": branch.request_id, "outcome": branch.outcome, "frontier": lineage.frontier, "terminal_kind": "never-created", "original_file": asdict(original)} + + +def _failed_branch_original(context: NativeNegativeContext) -> tuple[Path, dict[str, object]]: + """Reverify real L evidence and bind the fault to the issuer's saved input.""" + authorization = EpochAuthorization.parse(json.loads(read_original(AUTHORIZATION_FILE)[0])) + if authorization.target_id != context.enrollment.target.management_id: + raise AssertionError("the current authorization selects another target") + with ExitStack() as held: + descriptors = [] + for original in context.enrollment.originals: + owner = original.owner + if owner is None: + continue + _, identity = read_original(Path(owner.path)) + descriptor = os.open(owner.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + held.callback(os.close, descriptor) + if identity != (owner.device, owner.inode) or (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) != identity: + raise AssertionError("the actual original owner changed before independent lineage verification") + fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) + descriptors.append(descriptor) + history = held.enter_context(observe(authorization.history)) + current = derive_lineage(authorization.history, history, tuple(descriptors)) + verify_lineage(current, authorization.history, history, tuple(descriptors)) + intent = json.loads(read_original(context.epoch_root / "intent.json")[0]) + saved = json.loads(intent["binding"]["evidence_json"])["lineage"] + if canonical(asdict(current)) != canonical(saved): + raise AssertionError("the independently verified failed-branch scope differs from the actual retained intent") + path, observation = _failed_terminal(current) + observation.update(independent_lineage_verified=True, retained_intent_lineage_matched=True) + return path, observation + + def _mutation_file(context: NativeNegativeContext, scenario: str) -> Path: target = context.enrollment.target if scenario == "late-management-config": @@ -257,12 +370,6 @@ def _mutation_file(context: NativeNegativeContext, scenario: str) -> Path: authorization = EpochAuthorization.parse(json.loads(AUTHORIZATION_FILE.read_bytes())) if authorization.target_id != target.management_id: raise AssertionError("the actual fixed authorization selects a different target") - if scenario == "late-original-attempt": - path = Path(authorization.history.root) / "00000000.json" - original = json.loads(path.read_bytes()) - if original.get("kind") != "anchor": - raise AssertionError("the selected first event is not an actual original attempt") - return path if scenario == "late-ready-receipt": return Path(authorization.ready_receipt.path) if scenario == "late-installed-source": @@ -279,7 +386,9 @@ def mutate_after_intent(context: NativeNegativeContext, scenario: str, issuer_pi _require_publication_prefix(context, issuer_pid, "intent") observation: dict[str, object] = {"scenario": scenario, "issuer_pid": issuer_pid, "issuer_start_token": _token(issuer_pid), "boundary": "after-intent-head", "preservation_baseline": "after-deliberate-fixture-mutation"} with ExitStack() as held: - if scenario.endswith("-fence"): + if scenario == "late-target-loss": + observation["target_fault"] = held.enter_context(_remove_target(context)) + elif scenario.endswith("-fence"): pid = issuer_pid if scenario == "late-controller-fence": lock, mode = context.enrollment.watchers[0].lock, "WRITE" @@ -304,7 +413,10 @@ def mutate_after_intent(context: NativeNegativeContext, scenario: str, issuer_pi raise AssertionError("the real controller did not become startable without starting a process") observation["controller_fault"] = {"unit": unit, "before": asdict(before), "after": asdict(after)} else: - path = _mutation_file(context, scenario) + if scenario == "late-original-attempt": + path, observation["original_attempt"] = _failed_branch_original(context) + else: + path = _mutation_file(context, scenario) if not path.is_relative_to(context.fixture_root) or path != path.resolve(strict=True): raise PermissionError("late mutation escaped the separately authorized fixture root") before = _projection(path) @@ -557,6 +669,18 @@ def _descriptor_snapshot(pid: int) -> tuple[tuple[int, int, int, str], ...]: return tuple(result) +def _stopped_receiver(pid: int, start_token: str, pidfd: int, descriptors: tuple[tuple[int, int, int, str], ...]) -> dict[str, object]: + """Prove refusal preceded receiver death or any userspace acknowledgment.""" + poll = select.poll() + poll.register(pidfd, select.POLLIN) + if poll.poll(0) or _token(pid) != start_token: + raise AssertionError("the receiver lifetime ended before the original issuer refused") + state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] + if state not in {"T", "t"} or _descriptor_snapshot(pid) != descriptors: + raise AssertionError("the receipt-free receiver is not still stopped with its exact descriptors") + return {"pid": pid, "start_token": start_token, "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), "state": state, "descriptor_sha256": hashlib.sha256(canonical(descriptors)).hexdigest(), "pidfd_alive": True} + + def _assert_no_target_admission(context: NativeNegativeContext, *, running_pid: int | None = None) -> None: state = Path(context.enrollment.target.owner.path).parent config = BootstrapConfig.read(Path(context.enrollment.target.bootstrap_config.path)) @@ -610,6 +734,7 @@ def _run_peer_case(context: NativeNegativeContext, case: str) -> dict[str, objec target: DockerTraceProcess | None = None replacement: DockerTraceProcess | None = None paused_snapshot: tuple[tuple[int, int, int, str], ...] | None = None + unacknowledged_snapshot: tuple[tuple[int, int, int, str], ...] | None = None with ExitStack() as held: output = held.enter_context(log.open("xb")) try: @@ -644,8 +769,7 @@ def stop_receiver_before_offer() -> None: raise AssertionError("the successful native send did not precede actual receiver custody") observations["receiver_before_and_after_enqueue"] = paused_snapshot _assert_no_target_admission(context, running_pid=target.pid) - target.kill() - target.wait(30) + unacknowledged_snapshot = paused_snapshot host_barrier.release() elif receiving_offer: target = DockerTraceProcess(context) @@ -658,7 +782,7 @@ def stop_receiver_before_offer() -> None: observations["receiver_syscall"] = asdict(received) assert_unreserved_epoch(context.epoch_root) _assert_no_target_admission(context, running_pid=target.pid) - target_barrier.kill() + unacknowledged_snapshot = _descriptor_snapshot(target.pid) elif case == "changed-target-lifetime-refused": target = DockerTraceProcess(context) observations["original_target_lifetime"] = {"pid": target.pid, "start_token": target.start_token} @@ -680,6 +804,21 @@ def stop_receiver_before_offer() -> None: issuer.wait(timeout=120) if issuer.returncode != 2: raise AssertionError("the actual issuer did not refuse the incomplete native custody") + if unacknowledged_snapshot is not None: + assert target is not None + # Keep the exact live receiver blocked until the issuer's + # genuine receipt timeout refuses. Killing it beforehand would + # let a dead-peer check hide missing acknowledgment enforcement. + target.check() + observations["live_receiver_after_issuer_refusal"] = _stopped_receiver(target.pid, target.start_token, target.pidfd, unacknowledged_snapshot) + assert_unreserved_epoch(context.epoch_root) + _assert_no_target_admission(context, running_pid=target.pid) + observations.update(receiver_alive_until_issuer_refusal=True, issuer_refused_before_receiver_termination=True) + if target_barrier is not None: + target_barrier.kill() + else: + target.kill() + target.wait(30) _wait(lambda: _target_state(context).get("Running") is False, "the actual refused consumer exits", timeout=90) _assert_no_target_admission(context) if PEER_SOCKET.exists() or PEER_SOCKET.is_symlink(): diff --git a/scripts/recovery_vm/README.md b/scripts/recovery_vm/README.md index 8d1ecc908..e7b118640 100644 --- a/scripts/recovery_vm/README.md +++ b/scripts/recovery_vm/README.md @@ -27,6 +27,8 @@ before its first daemon start and verifies both imported images retain their exact approved IDs. Importing between these stores can change the meaning of Docker's image ID; a changed ID is a preparation failure, not a reason to select another image or weaken the production source checks. +Docker documents the [fresh-install storage default](https://docs.docker.com/engine/storage/containerd/) +and the [cross-store image-ID behavior](https://github.com/docker/cli/issues/6670). ## Reproduce a reviewed scenario @@ -61,7 +63,7 @@ any management fixture. It refuses a populated engine or earlier fixture roots. scenario oracle. CI derives its matrix directly from this list and requires every matrix job to pass. The two `kill-after-*` debug aliases select boundaries already present in the matrix and do not add independent cases. All before/after -publication cases and all ten mutations after the original intent head run on +publication cases and all eleven mutations after the original intent head run on independent fresh VMs. They use real ptrace syscall stops around the actual exclusive record/head creation and successful file/directory fsync, with a pinned SIGKILL and no syscall-result diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh index 542df0858..e07351b20 100644 --- a/scripts/recovery_vm/guest.sh +++ b/scripts/recovery_vm/guest.sh @@ -54,12 +54,16 @@ python3 -m venv /opt/fixture-tools mkdir -m 700 /opt/recovery-fixture /opt/reviewed-source tar --extract --gzip --file "$input/source.tar.gz" --directory /opt/recovery-fixture cd /opt/recovery-fixture -/opt/fixture-tools/bin/uv sync --locked --no-editable --no-dev --extra test +/opt/fixture-tools/bin/uv sync --locked --no-editable --no-dev --extra test --link-mode copy .venv/bin/python - <<'PY' from pathlib import Path import auto_coder.management.proxy_watch_cli as watcher -Path(watcher.__file__).chmod(0o600) +source = Path(watcher.__file__) +source.chmod(0o600) +identity = source.stat() +assert identity.st_uid == 0 and identity.st_nlink == 1 +print("Installed watcher source is a private, independently copied file.") PY docker compose version install -m 600 "$input/source-token" /run/recovery-fixture-source-token diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 49a345515..256066ff9 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -56,7 +56,9 @@ FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") RESULT_SCHEMA = "auto-coder-native-recovery-epoch-result-v1" ORIGINAL_NEGATIVE_CASES = frozenset({"original-journal-absence-refused", "changed-installed-ready-refused", "changed-target-management-config-refused"}) -LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused"}) +LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused", "cross-deployment-original-lineage-refused"}) +FACTORY_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES if case.startswith("factory-")) +TASK_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES | REQUIRED_REPORT_CASES if "nonleader" in case) ACCEPTANCE_SCENARIOS = ("grant", "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS, *LATE_MUTATION_SCENARIOS) SCENARIOS = (*ACCEPTANCE_SCENARIOS, "kill-after-intent", "kill-after-consume") REQUIRED_CASES = frozenset( @@ -204,6 +206,14 @@ def _no_target_child(target: Generation, *, running: bool) -> None: assert not Path(f"/proc/{pid}/task/{pid}/children").read_text().strip(), "prelaunch target has a child process" +def _require_target_absent(target_id: str) -> None: + """Prove exact object absence in the actual complete local inventory.""" + assert re.fullmatch(r"[0-9a-f]{64}", target_id), "the selected target must be an exact Docker ID" + identifiers = Chain.docker("ps", "--all", "--quiet", "--no-trunc").splitlines() + assert all(re.fullmatch(r"[0-9a-f]{64}", value) for value in identifiers) and len(set(identifiers)) == len(identifiers), "the complete Docker inventory is malformed" + assert target_id not in identifiers, "the deliberately removed target was recreated or remains present" + + def _native_refused_branch(chain: Chain, prior: Generation) -> Generation: candidate = chain.prepare("D", prior) assert candidate.intent is not None and candidate.proxy is not None @@ -276,6 +286,13 @@ def _check_lineage(chain: Chain, captures: tuple[CapturedGeneration, ...]) -> di pass else: raise AssertionError("a truncated original lineage was accepted as complete") + foreign = replace(chain.history_binding, deployment="foreign-deployment-" + uuid.uuid4().hex, history=str(uuid.uuid4())) + try: + verify_lineage(replace(complete, binding=foreign), chain.history_binding, history, tuple(descriptors)) + except OriginalEvidenceUnavailable: + pass + else: + raise AssertionError("a lineage report from another deployment/history was accepted") outcomes = {item.instance_id: item.outcome for item in complete.generations} assert set(outcomes) == {"A", "B", "C", "D"} assert outcomes["B"] == "succeeded" and outcomes["C"] in {"launched", "unknown"} and outcomes["D"] == "never-created" @@ -511,6 +528,15 @@ def inspect_durable_result(root: Path, *, issuer_log: str, consumer_log: str) -> } +def _observation_cases(value: dict[str, object], name: str, schema: str, expected: frozenset[str]) -> dict[str, dict[str, object]]: + document = value.get(name) + assert isinstance(document, dict) and document.get("schema") == schema + rows = document.get("cases") + assert isinstance(rows, list) and len(rows) == len(expected) and all(isinstance(item, dict) for item in rows) + assert {item.get("case") for item in rows} == expected + return {item["case"]: item for item in rows} + + def validate_result(value: dict[str, object]) -> None: """The CI oracle requires every native case and rejects partial acceptance.""" assert value.get("schema") == RESULT_SCHEMA and value.get("status") == "passed" @@ -522,13 +548,23 @@ def validate_result(value: dict[str, object]) -> None: assert isinstance(value.get("durable"), dict) if scenario == "grant": assert set(value["durable"]["record_sha256"]) == {"reserved", "intent", "consumed", "receipt"} - peer = value.get("peer_observations", {}) - assert peer.get("schema") == "actual-recovery-native-peer-faults-v1" - assert {item.get("case") for item in peer.get("cases", ())} == REQUIRED_PEER_CASES - for item in peer["cases"]: + peer = _observation_cases(value, "peer_observations", "actual-recovery-native-peer-faults-v1", REQUIRED_PEER_CASES) + for item in peer.values(): assert item.get("issuer_returncode") == 2 and item.get("epoch_unreserved") is True and item.get("issuer_endpoint_removed_by_original") is True assert item.get("consumer_grant_accepted") is False and item.get("no_child_or_admission") is True assert item.get("issuer_syscall", {}).get("bytes_observed", 0) > 0 + if item["case"] in {"queued-rights-without-receipt-refused", "received-rights-without-receipt-refused"}: + assert item.get("receiver_alive_until_issuer_refusal") is True and item.get("issuer_refused_before_receiver_termination") is True + factory = _observation_cases(value, "factory_override_observations", "actual-recovery-factory-overrides-v1", FACTORY_OBSERVATION_CASES) + for item in factory.values(): + assert item.get("returncode") == 2 and item.get("refusal") + assert re.fullmatch(r"[0-9a-f]{64}", item.get("alternate_inputs_sha256", "")) and re.fullmatch(r"[0-9a-f]{64}", item.get("epoch_sha256", "")) + tasks = _observation_cases(value, "task_observations", "actual-recovery-task-observations-v1", TASK_OBSERVATION_CASES) + assert tasks["escaped-nonleader-current-containment-refused"]["status"] == "unavailable" + exited = tasks["measured-exited-nonleader-current-containment-reverified"] + assert exited["status"] == "observed-current-recovery-containment" and exited["after_exit"] == exited["after_verification"] + assert exited["after_exit"]["pidfd_readable"] is True and exited["after_exit"]["nonleader_absent"] is True + assert exited["after_exit"]["leader"]["state"] == "Z" and exited["after_exit"]["leader"]["network_namespace"] is None elif scenario in BOUNDARY_SCENARIOS: publication, when = parse_boundary(scenario) assert value["durable"]["publication"] == publication and value["durable"]["when"] == when @@ -543,6 +579,17 @@ def validate_result(value: dict[str, object]) -> None: assert value.get("preservation_baseline") == "after-deliberate-fixture-mutation" assert value["durable"]["mutation"]["preservation_baseline"] == value["preservation_baseline"] assert value["preserved_originals_sha256"] == value["durable"]["originals_after_injection_sha256"] + if scenario == "late-target-loss": + target_fault = value["durable"]["mutation"]["target_fault"] + assert target_fault["pidfd_death_observed"] is True and target_fault["exact_docker_id_absent"] is True and target_fault["docker_rm_returncode"] == 0 + assert target_fault["other_docker_ids_preserved"] is True + elif scenario == "late-original-attempt": + attempt = value["durable"]["mutation"]["original_attempt"] + changed = value["durable"]["mutation"]["file_fault"] + assert (attempt["instance_id"], attempt["outcome"], attempt["terminal_kind"]) == ("D", "never-created", "never-created") + assert attempt["independent_lineage_verified"] is True and attempt["retained_intent_lineage_matched"] is True + assert attempt["original_file"]["path"] == changed["path"] and attempt["original_file"]["sha256"] == changed["before"][-1] + assert (attempt["original_file"]["device"], attempt["original_file"]["inode"]) == tuple(changed["before"][:2]) else: assert value["durable"]["phase"] == ("intent" if scenario == "kill-after-intent" else "consumed" if scenario == "kill-after-consume" else "receipt") assert value["durable"]["consumer_grant_acknowledged"] is False @@ -740,8 +787,11 @@ def _late_mutation( check_workers() barrier.release() assert issuer.wait(timeout=1800) == 2, "the actual issuer did not refuse the changed post-intent authority" - _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "late refusal ends the exact current consumer lifetime", timeout=120) - assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + if scenario == "late-target-loss": + _require_target_absent(target.generation.container) + else: + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "late refusal ends the exact current consumer lifetime", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" _no_target_child(target.generation, running=False) assert "Recovery custody acknowledged" not in consumer_log.read_text() and "One consumed recovery epoch grant send completed" not in issuer_log.read_text() assert _snapshot((EPOCH_ROOT,)) == epoch, "late mutation reached consumption or rewrote original epoch evidence" @@ -749,6 +799,8 @@ def _late_mutation( check_workers() replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == epoch, "the uncertain original intent was repaired or reissued" + if scenario == "late-target-loss": + _require_target_absent(target.generation.container) assert _snapshot(retained_roots) == injected check_workers() result = { @@ -866,6 +918,7 @@ def preservation() -> object: cases.extend(LINEAGE_REPORT_CASES) cases.extend(run_native_negatives(native_context, lambda label: _issuer_refusal(native_context, target, label), preservation)) + result["factory_override_observations"] = json.loads((root / "native-factory-override-observations.json").read_bytes()) cases.extend(_original_negatives(native_context, target, chain, preservation)) cases.extend(run_native_peer_cases(native_context, preservation)) result["peer_observations"] = json.loads((root / "native-peer-observations.json").read_bytes()) @@ -893,6 +946,7 @@ def preservation() -> object: cases.append("genuine-installed-ready-in-target") if scenario == "grant": cases.extend(verify_native_containment_reports(native_context, issuer.pid)) + result["task_observations"] = json.loads((root / "native-task-observations.json").read_bytes()) assert _snapshot(retained_roots) == preserved workers.preserved(*worker_before) consumer_log = target.generation.path("state") / "logs" / "recovery-epoch-bootstrap.log" diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index e652fff3d..3140694e8 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -106,6 +106,25 @@ def test_no_child_or_admission_oracle_rejects_any_native_store(tmp_path: Path) - fixture._no_target_child(generation, running=False) +@pytest.mark.parametrize("inventory", ["a" * 64, "truncated-id", "\n".join(["b" * 64, "b" * 64])]) +def test_removed_target_oracle_rejects_presence_or_incomplete_inventory(monkeypatch: pytest.MonkeyPatch, inventory: str) -> None: + monkeypatch.setattr(fixture.Chain, "docker", lambda *_args: inventory) + with pytest.raises(AssertionError): + fixture._require_target_absent("a" * 64) + + +def test_removed_target_oracle_compares_exact_full_ids(monkeypatch: pytest.MonkeyPatch) -> None: + calls = [] + + def docker(*arguments): + calls.append(arguments) + return "a" * 63 + "b" + + monkeypatch.setattr(fixture.Chain, "docker", docker) + fixture._require_target_absent("a" * 64) + assert calls == [("ps", "--all", "--quiet", "--no-trunc")] + + def test_original_fingerprint_binds_inodes_and_bytes(tmp_path: Path) -> None: path = tmp_path / "native-original" path.write_bytes(b"original") @@ -204,7 +223,37 @@ def _result() -> dict[str, object]: "durable": {"record_sha256": {name: "a" * 64 for name in ("reserved", "intent", "consumed", "receipt")}}, "peer_observations": { "schema": "actual-recovery-native-peer-faults-v1", - "cases": [{"case": case, "issuer_returncode": 2, "epoch_unreserved": True, "issuer_endpoint_removed_by_original": True, "consumer_grant_accepted": False, "no_child_or_admission": True, "issuer_syscall": {"bytes_observed": 10}} for case in sorted(fixture.REQUIRED_PEER_CASES)], + "cases": [ + { + "case": case, + "issuer_returncode": 2, + "epoch_unreserved": True, + "issuer_endpoint_removed_by_original": True, + "consumer_grant_accepted": False, + "no_child_or_admission": True, + "issuer_syscall": {"bytes_observed": 10}, + "receiver_alive_until_issuer_refusal": True, + "issuer_refused_before_receiver_termination": True, + } + for case in sorted(fixture.REQUIRED_PEER_CASES) + ], + }, + "factory_override_observations": { + "schema": "actual-recovery-factory-overrides-v1", + "cases": [{"case": case, "returncode": 2, "refusal": "component refusal", "alternate_inputs_sha256": "d" * 64, "epoch_sha256": "e" * 64} for case in sorted(fixture.FACTORY_OBSERVATION_CASES)], + }, + "task_observations": { + "schema": "actual-recovery-task-observations-v1", + "cases": [ + {"case": "escaped-old-network-nonleader-refused"}, + {"case": "escaped-nonleader-current-containment-refused", "status": "unavailable"}, + { + "case": "measured-exited-nonleader-current-containment-reverified", + "status": "observed-current-recovery-containment", + "after_exit": {"pidfd_readable": True, "nonleader_absent": True, "leader": {"state": "Z", "network_namespace": None}}, + "after_verification": {"pidfd_readable": True, "nonleader_absent": True, "leader": {"state": "Z", "network_namespace": None}}, + }, + ], }, } @@ -272,6 +321,35 @@ def test_grant_result_requires_the_actual_peer_evidence_inventory(case: str) -> fixture.validate_result(result) +@pytest.mark.parametrize("field", ["receiver_alive_until_issuer_refusal", "issuer_refused_before_receiver_termination"]) +@pytest.mark.parametrize("case", ["queued-rights-without-receipt-refused", "received-rights-without-receipt-refused"]) +def test_missing_peer_receipt_cannot_be_masked_by_an_already_dead_receiver(case: str, field: str) -> None: + result = _result() + fixture.validate_result(result) + selected = next(item for item in result["peer_observations"]["cases"] if item["case"] == case) + selected[field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("name", ["factory_override_observations", "task_observations"]) +def test_native_factory_and_task_measurements_are_required(name: str) -> None: + result = _result() + fixture.validate_result(result) + result.pop(name) + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +def test_exited_task_success_requires_the_same_observed_zombie() -> None: + result = _result() + fixture.validate_result(result) + selected = next(item for item in result["task_observations"]["cases"] if item["case"] == "measured-exited-nonleader-current-containment-reverified") + selected["after_verification"]["leader"]["state"] = "R" + with pytest.raises(AssertionError): + fixture.validate_result(result) + + @pytest.mark.parametrize("scenario", fixture.LATE_MUTATION_SCENARIOS) def test_late_mutation_result_names_its_deliberate_preservation_boundary(scenario: str) -> None: result = _result() @@ -282,6 +360,13 @@ def test_late_mutation_result_names_its_deliberate_preservation_boundary(scenari preserved_originals_sha256="c" * 64, durable={"phase": "intent", "consumer_grant_acknowledged": False, "grant_send_completed": False, "mutation": {"scenario": scenario, "boundary": "after-intent-head", "preservation_baseline": "after-deliberate-fixture-mutation"}, "originals_after_injection_sha256": "c" * 64}, ) + if scenario == "late-target-loss": + result["durable"]["mutation"]["target_fault"] = {"pidfd_death_observed": True, "exact_docker_id_absent": True, "docker_rm_returncode": 0, "other_docker_ids_preserved": True} + elif scenario == "late-original-attempt": + result["durable"]["mutation"].update( + original_attempt={"instance_id": "D", "outcome": "never-created", "terminal_kind": "never-created", "independent_lineage_verified": True, "retained_intent_lineage_matched": True, "original_file": {"path": "/component/history/00000007.json", "device": 1, "inode": 2, "sha256": "a" * 64}}, + file_fault={"path": "/component/history/00000007.json", "before": [1, 2, "a" * 64]}, + ) fixture.validate_result(result) result["preservation_baseline"] = "before-deliberate-fixture-mutation" with pytest.raises(AssertionError): @@ -292,6 +377,29 @@ def test_late_mutation_result_names_its_deliberate_preservation_boundary(scenari fixture.validate_result(result) +@pytest.mark.parametrize("field", ["pidfd_death_observed", "exact_docker_id_absent", "other_docker_ids_preserved"]) +def test_late_target_loss_requires_death_absence_and_preserved_other_objects(field: str) -> None: + scenario = "late-target-loss" + result = _result() + result.update( + scenario=scenario, + cases=sorted(fixture.required_cases(scenario)), + preservation_baseline="after-deliberate-fixture-mutation", + preserved_originals_sha256="c" * 64, + durable={ + "phase": "intent", + "consumer_grant_acknowledged": False, + "grant_send_completed": False, + "mutation": {"scenario": scenario, "boundary": "after-intent-head", "preservation_baseline": "after-deliberate-fixture-mutation", "target_fault": {"pidfd_death_observed": True, "exact_docker_id_absent": True, "docker_rm_returncode": 0, "other_docker_ids_preserved": True}}, + "originals_after_injection_sha256": "c" * 64, + }, + ) + fixture.validate_result(result) + result["durable"]["mutation"]["target_fault"][field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + @pytest.mark.parametrize("scenario", fixture.BOUNDARY_SCENARIOS) def test_publication_result_requires_exact_kernel_observation(scenario: str) -> None: publication, when = fixture.parse_boundary(scenario) diff --git a/tests/test_management_recovery_epoch_native_negatives.py b/tests/test_management_recovery_epoch_native_negatives.py index 32c9cb155..2fffbd876 100644 --- a/tests/test_management_recovery_epoch_native_negatives.py +++ b/tests/test_management_recovery_epoch_native_negatives.py @@ -7,7 +7,9 @@ from __future__ import annotations import fcntl +import json import os +import select import signal import stat import subprocess @@ -125,6 +127,65 @@ def test_modified_injection_is_retained_instead_of_destructively_restored(tmp_pa assert fixture._projection(backup) == original +def test_missing_owner_injection_never_creates_a_substitute_and_restores_the_original(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = tmp_path / "owner.lock", tmp_path / "owner.original" + _write_private(path, b"original owner") + before = fixture._projection(path) + with fixture._missing_original(context.epoch_root, path, backup): + assert not path.exists() + assert fixture._projection(backup) == before + fixture.assert_unreserved_epoch(context.epoch_root) + assert fixture._projection(path) == before + assert not backup.exists() + + +def test_issuer_recreated_missing_owner_is_retained_as_a_failure(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = tmp_path / "owner.lock", tmp_path / "owner.original" + _write_private(path, b"original owner") + before = fixture._projection(path) + with pytest.raises(AssertionError, match="recreated"): + with fixture._missing_original(context.epoch_root, path, backup): + _write_private(path, b"unexpected replacement") + assert fixture._projection(backup) == before + assert path.read_bytes() == b"unexpected replacement" + assert path.stat().st_ino != backup.stat().st_ino + + +def test_publication_during_missing_owner_case_never_restores_or_retries(tmp_path: Path) -> None: + context = _context(tmp_path) + path, backup = tmp_path / "owner.lock", tmp_path / "owner.original" + _write_private(path, b"original owner") + before = fixture._projection(path) + with pytest.raises(fixture.EpochAlreadyPublished, match="never repair or retry"): + with fixture._missing_original(context.epoch_root, path, backup): + _write_private(context.epoch_root / "reserved.json", b"uncertain original publication") + assert not path.exists() + assert fixture._projection(backup) == before + assert (context.epoch_root / "reserved.json").read_bytes() == b"uncertain original publication" + + +@pytest.mark.parametrize("change", ["contents", "inode", "added-entry", "removed-entry"]) +def test_alternate_authority_snapshot_detects_real_file_or_membership_changes(tmp_path: Path, change: str) -> None: + root = tmp_path / "alternate" + root.mkdir(mode=0o700) + path = root / "inventory.json" + _write_private(path, b"original alternate input") + before = fixture._alternate_input_snapshot(root) + assert fixture._alternate_input_snapshot(root) == before + if change == "contents": + path.write_bytes(b"modified alternate input") + elif change == "inode": + path.rename(tmp_path / "retained-original") + _write_private(path, b"original alternate input") + elif change == "added-entry": + _write_private(root / "reserved.json", b"unexpected publication") + else: + path.unlink() + assert fixture._alternate_input_snapshot(root) != before + + def test_each_refusal_compares_injected_and_restored_original_state(tmp_path: Path) -> None: context = _context(tmp_path) original, backup = tmp_path / "owner", tmp_path / "owner.original" @@ -203,6 +264,45 @@ def test_malformed_real_process_readiness_is_a_failure() -> None: pytest.fail("an invalid native barrier was accepted") +def test_exited_nonleader_control_retains_the_same_real_zombie_for_an_independent_reader() -> None: + code = "import json,threading;event=threading.Event();\ndef occupy():\n print(json.dumps({'tid':threading.get_native_id()}),flush=True);event.wait()\nthread=threading.Thread(target=occupy);thread.start();thread.join()" + process = subprocess.Popen((sys.executable, "-I", "-B", "-c", code), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + descriptor = os.pidfd_open(process.pid, 0) + try: + assert process.stdout is not None and select.select((process.stdout,), (), (), 10)[0] + tid = json.loads(process.stdout.readline())["tid"] + leader = fixture._task_observation(process.pid, process.pid) + nonleader = fixture._task_observation(process.pid, tid) + assert leader.pid == nonleader.pid and leader.tid != nonleader.tid + assert leader.network_namespace == nonleader.network_namespace == os.readlink("/proc/self/ns/net") + assert leader.state not in {"Z", "X", "x"} and nonleader.state not in {"Z", "X", "x"} + signal.pidfd_send_signal(descriptor, signal.SIGTERM) + exited = fixture._wait_unreaped_exit(process.pid, descriptor, leader.start_token, tid) + assert exited.leader.start_token == leader.start_token and exited.leader.state == "Z" + assert exited.leader.network_namespace is None and exited.nonleader_absent + assert exited.remaining_tids == (process.pid,) and exited.pidfd_readable + assert (exited.wait_code, exited.wait_status) == (os.CLD_KILLED, signal.SIGTERM) + probe = ( + "import json,os,sys;from pathlib import Path;p=Path('/proc')/sys.argv[1];s=(p/'stat').read_text().rsplit(')',1)[1].split();" + "assert s[0]=='Z' and s[19]==sys.argv[2];assert sorted(x.name for x in (p/'task').iterdir())==[sys.argv[1]];\n" + "try: os.readlink(p/'ns/net')\nexcept FileNotFoundError: print(json.dumps({'state':s[0],'start':s[19],'namespace':None}))\n" + "else: raise AssertionError('exited task retained nsproxy')" + ) + observed = subprocess.run((sys.executable, "-I", "-B", "-c", probe, str(process.pid), leader.start_token), capture_output=True, text=True, timeout=10, check=True) + assert json.loads(observed.stdout) == {"state": "Z", "start": leader.start_token, "namespace": None} + assert fixture._wait_unreaped_exit(process.pid, descriptor, leader.start_token, tid) == exited + with pytest.raises(AssertionError, match="measured namespace-free zombie"): + fixture._wait_unreaped_exit(process.pid, descriptor, str(int(leader.start_token) + 1), tid) + assert process.wait(timeout=10) == -signal.SIGTERM + with pytest.raises(ChildProcessError): + fixture._wait_unreaped_exit(process.pid, descriptor, leader.start_token, tid) + finally: + if process.poll() is None: + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + process.communicate(timeout=10) + os.close(descriptor) + + _RESOURCE_HOLDER = """ import fcntl import os @@ -270,9 +370,10 @@ def test_native_entry_refuses_unapproved_roots_before_any_fixture_effect(tmp_pat def test_native_case_oracles_distinguish_actual_issuer_and_separate_report_work() -> None: - assert len(fixture.REQUIRED_NEGATIVE_CASES) == 9 - assert len(fixture.REQUIRED_REPORT_CASES) == 4 + assert len(fixture.REQUIRED_NEGATIVE_CASES) == 17 + assert len(fixture.REQUIRED_REPORT_CASES) == 6 assert fixture.REQUIRED_NEGATIVE_CASES.isdisjoint(fixture.REQUIRED_REPORT_CASES) - assert {"busy-original-owner-refused", "busy-controller-fence-refused", "replaced-epoch-lock-refused", "retained-old-network-task-and-listener-refused", "old-published-listener-refused"} <= fixture.REQUIRED_NEGATIVE_CASES - assert {"forged-native-containment-refused", "cross-deployment-containment-refused", "stale-native-containment-refused"} <= fixture.REQUIRED_REPORT_CASES + assert {"busy-original-owner-refused", "busy-controller-fence-refused", "missing-original-owner-refused", "replaced-epoch-lock-refused", "retained-old-network-task-and-listener-refused", "escaped-old-network-nonleader-refused", "old-published-listener-refused"} <= fixture.REQUIRED_NEGATIVE_CASES + assert {"forged-native-containment-refused", "cross-deployment-containment-refused", "stale-native-containment-refused", "escaped-nonleader-current-containment-refused", "measured-exited-nonleader-current-containment-reverified"} <= fixture.REQUIRED_REPORT_CASES + assert {"factory-root-argument-refused", "factory-inventory-argument-refused", "factory-epoch-root-environment-refused", "factory-inventory-environment-refused", "factory-authorization-environment-refused", "factory-containment-environment-refused"} <= fixture.REQUIRED_NEGATIVE_CASES assert stat.S_IMODE(Path(fixture.__file__).stat().st_mode) & 0o111 == 0 diff --git a/tests/test_management_recovery_epoch_native_peer.py b/tests/test_management_recovery_epoch_native_peer.py index 7bb90b104..93372ab1a 100644 --- a/tests/test_management_recovery_epoch_native_peer.py +++ b/tests/test_management_recovery_epoch_native_peer.py @@ -12,7 +12,7 @@ import os import socket from contextlib import ExitStack, contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path import pytest @@ -20,13 +20,20 @@ LATE_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, KernelPeerMessageBarrier, + _container_ids, _ControlHeader, _decode_rights, + _descriptor_snapshot, + _failed_terminal, _Iovec, _MessageHeader, + _stopped_receiver, + _verify_target_loss, mutate_after_intent, ) +from auto_coder.management.original_attempt_evidence import OriginalFile, OriginalGeneration, OriginalLineageEvidence +from auto_coder.management.original_attempts import HistoryBinding, canonical from auto_coder.management.recovery_epoch_peer import MAX_MESSAGE_BYTES, MAX_RESOURCES, OFFER_SCHEMA, encode_peer_message pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") @@ -114,7 +121,7 @@ def test_late_mutation_selector_refuses_unreviewed_actions_before_access(scenari def test_native_case_sets_are_explicit_and_distinct() -> None: assert len(REQUIRED_PEER_CASES) == 4 - assert len(LATE_MUTATION_SCENARIOS) == 10 and len(set(LATE_MUTATION_SCENARIOS)) == 10 + assert len(LATE_MUTATION_SCENARIOS) == 11 and len(set(LATE_MUTATION_SCENARIOS)) == 11 assert not REQUIRED_PEER_CASES.intersection(LATE_MUTATION_SCENARIOS) @@ -170,3 +177,173 @@ def test_real_shared_ofd_reacquisition_changes_kernel_acquirer_visible_to_origin if process.poll() is None: process.kill() process.communicate(timeout=10) + + +@pytest.mark.parametrize("encoded", ["a" * 12, "A" * 64, "a" * 64 + "\n" + "a" * 64, "name " + "a" * 64]) +def test_target_loss_inventory_rejects_truncated_alias_or_duplicate_identities(encoded: str) -> None: + with pytest.raises(AssertionError, match="incomplete or conflicting"): + _container_ids(encoded) + + +def test_target_loss_inventory_preserves_complete_ids_and_accepts_measured_empty_output() -> None: + assert _container_ids("b" * 64 + "\n" + "a" * 64 + "\n") == ("a" * 64, "b" * 64) + assert _container_ids("") == () + + +def test_target_loss_oracle_requires_real_pidfd_death_and_exact_object_absence() -> None: + """Exercise kernel lifetime proof; synthetic IDs are not Docker acceptance.""" + import select + import signal + import subprocess + import sys + + process = subprocess.Popen([sys.executable, "-I", "-B", "-c", "import signal; print('alive', flush=True); signal.pause()"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + descriptor = os.pidfd_open(process.pid, 0) + try: + assert select.select((process.stdout,), (), (), 10)[0] + assert process.stdout.readline() == "alive\n" + identifier, other = "a" * 64, "b" * 64 + with pytest.raises(AssertionError, match="pidfd has not independently"): + _verify_target_loss(descriptor, identifier, (identifier, other), (other,)) + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + poll = select.poll() + poll.register(descriptor, select.POLLIN) + assert poll.poll(10000), "the actual subprocess death was not observed" + _verify_target_loss(descriptor, identifier, (identifier, other), (other,)) + for after in ((identifier, other), (), ("c" * 64,)): + with pytest.raises(AssertionError, match="exact target was not removed"): + _verify_target_loss(descriptor, identifier, (identifier, other), after) + assert process.wait(timeout=10) == -signal.SIGKILL + finally: + if process.poll() is None: + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + process.communicate(timeout=10) + os.close(descriptor) + + +def _component_failed_lineage(root: Path) -> tuple[OriginalLineageEvidence, Path]: + """Use explicit synthetic records only for the selector's component oracle.""" + import hashlib + + request = "67bb23bb-35e4-45dd-b9dc-40ba21f912fb" + terminal = root / "00000007.json" + values = ( + (root / "00000000.json", canonical({"kind": "anchor", "attempt": "other-original-attempt"})), + (terminal, canonical({"kind": "never-created", "attempt": request, "value": {}})), + (root / "history.lock", b""), + ) + originals = [] + for path, encoded in values: + path.write_bytes(encoded) + path.chmod(0o600) + info = path.stat() + originals.append(OriginalFile(str(path), info.st_dev, info.st_ino, hashlib.sha256(encoded).hexdigest())) + branch = OriginalGeneration( + request_id=request, + instance_id="D", + management_id="d" * 64, + image_id="sha256:" + "e" * 64, + bootstrap_config_sha256="b" * 64, + management_config_sha256="c" * 64, + state_root=str(root / "state"), + state_device=1, + state_inode=2, + journal_root=str(root / "journal"), + journal_device=1, + journal_inode=3, + oauth_root=str(root / "oauth"), + oauth_device=1, + oauth_inode=4, + owner_path="", + owner_device=0, + owner_inode=0, + outcome="never-created", + expected_stores=(), + ) + return OriginalLineageEvidence(HistoryBinding(str(root), "component-deployment", "component-history", "sha256:" + "a" * 64), frontier="f" * 64, generations=(branch,), originals=tuple(originals)), terminal + + +def test_failed_branch_selector_uses_matched_terminal_original_not_first_history_path(tmp_path: Path) -> None: + lineage, terminal = _component_failed_lineage(tmp_path) + selected, observation = _failed_terminal(lineage) + assert selected == terminal and selected.name != "00000000.json" + assert observation["instance_id"] == "D" and observation["outcome"] == "never-created" + assert observation["request_id"] == lineage.generations[0].request_id + assert observation["frontier"] == lineage.frontier + assert observation["original_file"]["inode"] == terminal.stat().st_ino + assert "independent_lineage_verified" not in observation, "a pure selector must never claim that the producer ran" + + +@pytest.mark.parametrize("fault", ["absent", "changed-bytes", "wrong-request", "duplicate-record", "duplicate-branch", "unknown", "incomplete"]) +def test_failed_branch_selector_refuses_missing_changed_or_ambiguous_originals(tmp_path: Path, fault: str) -> None: + import hashlib + import json + + lineage, terminal = _component_failed_lineage(tmp_path) + selected = next(original for original in lineage.originals if original.path == str(terminal)) + if fault == "absent": + lineage = replace(lineage, originals=tuple(original for original in lineage.originals if original != selected)) + elif fault == "changed-bytes": + terminal.write_bytes(terminal.read_bytes() + b"\x00") + elif fault == "wrong-request": + value = json.loads(terminal.read_bytes()) + value["attempt"] = "different-failed-request" + encoded = canonical(value) + terminal.write_bytes(encoded) + lineage = replace(lineage, originals=tuple(replace(original, sha256=hashlib.sha256(encoded).hexdigest()) if original == selected else original for original in lineage.originals)) + elif fault == "duplicate-record": + duplicate = terminal.with_name("00000008.json") + duplicate.write_bytes(terminal.read_bytes()) + duplicate.chmod(0o600) + info = duplicate.stat() + lineage = replace(lineage, originals=(*lineage.originals, replace(selected, path=str(duplicate), device=info.st_dev, inode=info.st_ino))) + elif fault == "duplicate-branch": + lineage = replace(lineage, generations=(*lineage.generations, replace(lineage.generations[0], instance_id="another-failure"))) + elif fault == "unknown": + lineage = replace(lineage, generations=(replace(lineage.generations[0], outcome="unknown"),)) + else: + lineage = replace(lineage, unavailable=("an expected original is missing",)) + with pytest.raises(AssertionError, match="failed.branch|terminal record"): + _failed_terminal(lineage) + + +def test_live_stopped_receiver_oracle_cannot_count_running_or_dead_peer_as_unacknowledged_custody() -> None: + """Observe an actual stopped process; this is not native Docker acceptance.""" + import select + import signal + import subprocess + import sys + import time + + from scripts.recovery_epoch_fault_tracer import _token + + process = subprocess.Popen([sys.executable, "-I", "-B", "-c", "import signal; print('alive', flush=True); signal.pause()"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + descriptor = os.pidfd_open(process.pid, 0) + try: + assert select.select((process.stdout,), (), (), 10)[0] + assert process.stdout.readline() == "alive\n" + token = _token(process.pid) + snapshot = _descriptor_snapshot(process.pid) + with pytest.raises(AssertionError, match="not still stopped"): + _stopped_receiver(process.pid, token, descriptor, snapshot) + signal.pidfd_send_signal(descriptor, signal.SIGSTOP) + deadline = time.monotonic() + 10 + while Path(f"/proc/{process.pid}/stat").read_text().rsplit(")", 1)[1].split()[0] not in {"T", "t"}: + assert time.monotonic() < deadline + time.sleep(0.01) + observation = _stopped_receiver(process.pid, token, descriptor, snapshot) + assert observation["pidfd_alive"] is True and observation["pid"] == process.pid and observation["start_token"] == token + with pytest.raises(AssertionError, match="not still stopped"): + _stopped_receiver(process.pid, token, descriptor, snapshot[:-1]) + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + poll = select.poll() + poll.register(descriptor, select.POLLIN) + assert poll.poll(10000) + with pytest.raises(AssertionError, match="lifetime ended"): + _stopped_receiver(process.pid, token, descriptor, snapshot) + assert process.wait(timeout=10) == -signal.SIGKILL + finally: + if process.poll() is None: + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + process.communicate(timeout=10) + os.close(descriptor) From de0c156cd342a7f294c80d4e93f0b7d947fc4706 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 01:57:18 +0900 Subject: [PATCH 07/24] fix(management): preserve strict recovery fixture profiles --- .../workflows/management-recovery-epoch.yml | 4 +- .../management-original-genesis.md | 17 +- scripts/recovery_epoch_native_peer.py | 172 +++++++++++++++++- .../test_management_recovery_epoch_host.py | 31 +++- .../management/fresh_host_observation.py | 15 -- .../management/recovery_containment.py | 9 +- .../recovery_containment_profile.py | 3 + ...t_management_archived_cutover_container.py | 13 +- ...est_management_original_genesis_capture.py | 19 +- tests/test_management_recovery_containment.py | 3 +- ...nagement_recovery_containment_resources.py | 134 +++++++++++++- tests/test_management_recovery_epoch_host.py | 93 +++++++++- ...t_management_recovery_epoch_native_peer.py | 66 +++++++ ...test_management_recovery_proxy_profiles.py | 76 ++++++++ 14 files changed, 596 insertions(+), 59 deletions(-) create mode 100644 tests/test_management_recovery_proxy_profiles.py diff --git a/.github/workflows/management-recovery-epoch.yml b/.github/workflows/management-recovery-epoch.yml index d243b4287..319d2305f 100644 --- a/.github/workflows/management-recovery-epoch.yml +++ b/.github/workflows/management-recovery-epoch.yml @@ -30,7 +30,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 with: - version: "0.13.6" + version: "0.7.22" - name: Install locked test dependencies run: uv sync --locked --dev --extra test - name: Select every independent production scenario @@ -75,7 +75,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v7 with: - version: "0.13.6" + version: "0.7.22" - name: Install locked test dependencies run: uv sync --locked --dev --extra test - name: Inspect the actual guest acceleration device diff --git a/docs/client-features/management-original-genesis.md b/docs/client-features/management-original-genesis.md index 729496b8c..2a81b6c6c 100644 --- a/docs/client-features/management-original-genesis.md +++ b/docs/client-features/management-original-genesis.md @@ -70,12 +70,15 @@ executable receipt, the installed management console wrapper and its supported environment. A missing original root or an executable environment hook refuses without reconstructing the missing state. -The ordinary proxy may use its explicit owner-only writable `/data` and `/config` -bind roots as a pair. The ordinary predecessor capture checks their exact -canonical host directories; anonymous volumes or a lone additional data mount -do not meet this profile. A genesis predecessor carries the same read-only -`/management-original-history` mount into the existing successor capture, whose -host reader verifies that binding against the original root authority. +The existing ordinary successor capture keeps its strict single Caddyfile bind +profile. The integrated genesis fixture copies the verified Caddy binary from +its exact imported source image into an image without inherited data volumes, +then creates the proxy with `/srv` as its working directory and only the +read-only Caddyfile bind. Extra `/data` or `/config` mounts are not admitted by +that existing capture path. Stock Caddy's separate current-containment capture +does not relax this contract. A genesis predecessor carries the same read-only +`/management-original-history` mount into successor capture, whose host reader +verifies that binding against the original root authority. Root authorization JSON contains exactly the fields of `GenesisRootAuthorization`: `profile` (`ordinary-original-roots-v1`), `purpose` @@ -189,7 +192,7 @@ these tests are not real-Docker, first-child or recovery-epoch acceptance. against a missing original state root and proves that logger setup does not recreate it. The component selection also checks effective configuration mismatches and executable/environment failures before native publication. The additive -predecessor and explicit proxy-data capture checks are exercised in +predecessor binding and strict proxy mount checks are exercised in `tests/test_management_original_genesis_capture.py`. The production interfaces are suitable for isolated real-Docker and host-root diff --git a/scripts/recovery_epoch_native_peer.py b/scripts/recovery_epoch_native_peer.py index 24dca67f4..323d9f79c 100644 --- a/scripts/recovery_epoch_native_peer.py +++ b/scripts/recovery_epoch_native_peer.py @@ -19,6 +19,7 @@ import signal import socket import stat +import struct import subprocess import sys import time @@ -440,6 +441,173 @@ def mutate_after_intent(context: NativeNegativeContext, scenario: str, issuer_pi raise AssertionError("production refusal or replay changed the deliberately unmasked controller") +def _socket_option(descriptor: int, option: int, size: int) -> bytes: + """Read kernel socket metadata without constructing or reconfiguring a socket.""" + libc = ctypes.CDLL(None, use_errno=True) + libc.getsockopt.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint)) + libc.getsockopt.restype = ctypes.c_int + value = ctypes.create_string_buffer(size) + length = ctypes.c_uint(size) + result = libc.getsockopt(descriptor, socket.SOL_SOCKET, option, value, ctypes.byref(length)) + if result < 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + if length.value != size: + raise AssertionError("the actual socket metadata has an incomplete kernel representation") + return value.raw + + +def _local_socket_peer(descriptor: int) -> dict[str, int] | None: + """Return the real AF_UNIX stream peer's host-visible credentials.""" + domain = struct.unpack("i", _socket_option(descriptor, socket.SO_DOMAIN, struct.calcsize("i")))[0] + kind = struct.unpack("i", _socket_option(descriptor, socket.SO_TYPE, struct.calcsize("i")))[0] + if domain != socket.AF_UNIX or kind != socket.SOCK_STREAM: + return None + pid, uid, gid = struct.unpack("iII", _socket_option(descriptor, socket.SO_PEERCRED, struct.calcsize("iII"))) + return {"pid": pid, "uid": uid, "gid": gid} + + +def _peek_queued_packet(descriptor: int, expected: bytes) -> tuple[bytes, int, int]: + """Observe the actual queued frame twice, without consuming or changing flags.""" + if not expected.endswith(b"\n") or not 0 < len(expected) <= MAX_MESSAGE_BYTES: + raise AssertionError("a bounded complete expected grant frame is required") + libc = ctypes.CDLL(None, use_errno=True) + libc.recv.argtypes = (ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int) + libc.recv.restype = ctypes.c_ssize_t + before = fcntl.fcntl(descriptor, fcntl.F_GETFL) + + def peek() -> bytes: + buffer = ctypes.create_string_buffer(MAX_MESSAGE_BYTES + 1) + result = libc.recv(descriptor, buffer, len(buffer), socket.MSG_PEEK | socket.MSG_DONTWAIT) + if result < 0: + error = ctypes.get_errno() + raise AssertionError("the actual receiver has no observable complete queued grant: " + os.strerror(error)) + if result <= 0 or result > MAX_MESSAGE_BYTES: + raise AssertionError("the actual receiver has an empty or oversized queued grant") + return buffer.raw[:result] + + encoded = peek() + if encoded != expected: + raise AssertionError("the actual queued bytes differ from the exact consumed grant") + if peek() != encoded: + raise AssertionError("the actual grant bytes did not remain queued after MSG_PEEK") + after = fcntl.fcntl(descriptor, fcntl.F_GETFL) + if after != before: + raise AssertionError("observing the actual grant changed shared socket status flags") + return encoded, before, after + + +def _expected_queued_grant(context: NativeNegativeContext, issuer_pid: int, issuer_start_token: str, target_pid: int, target_start_token: str) -> dict[str, object]: + """Bind the expected packet to fixed consumer inputs and original durable custody.""" + configuration = _consumer_configuration(context) + inventory_bytes = read_original(context.epoch_root / "inventory.json")[0] + consumed_bytes = read_original(context.epoch_root / "consumed.json")[0] + receipt_bytes = read_original(context.epoch_root / "receipt.json")[0] + receipt = json.loads(receipt_bytes) + binding = receipt["binding"] + evidence_bytes = binding["evidence_json"].encode() + evidence = json.loads(evidence_bytes) + if canonical(evidence) != evidence_bytes or hashlib.sha256(evidence_bytes).hexdigest() != binding["evidence_sha256"]: + raise AssertionError("the actual retained custody evidence is not canonical and digest-bound") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + if (binding["host_pid"], binding["host_start_token"], binding["host_boot_id"]) != (issuer_pid, issuer_start_token, boot): + raise AssertionError("the original consumed evidence names another issuer lifetime") + custody = evidence["custody"] + peer = custody["peer"] + if (peer["pid"], peer["start_token"], peer["boot_id"], peer["uid"]) != (target_pid, target_start_token, boot, 0): + raise AssertionError("the original consumed custody names another target lifetime") + selected = {name: getattr(configuration, name) for name in ("request_id", "authorization_id", "target_id")} + if any(binding[name] != value or peer[name] != value for name, value in selected.items()): + raise AssertionError("the original consumed custody differs from the actual fixed consumer binding") + for name in ("challenge", "receipt_nonce"): + if not isinstance(custody[name], str) or re.fullmatch(r"[0-9a-f]{64}", custody[name]) is None: + raise AssertionError("the actual original custody challenge or receipt nonce is incomplete") + return { + "schema": GRANT_SCHEMA, + "granted": "recovery-epoch-consumed", + **selected, + "challenge": custody["challenge"], + "receipt_nonce": custody["receipt_nonce"], + "inventory_sha256": hashlib.sha256(inventory_bytes).hexdigest(), + "binding_sha256": hashlib.sha256(canonical(binding)).hexdigest(), + "consumed_sha256": hashlib.sha256(consumed_bytes).hexdigest(), + "receipt_sha256": hashlib.sha256(receipt_bytes).hexdigest(), + "host_boot_id": boot, + "host_pid": issuer_pid, + "host_start_token": issuer_start_token, + } + + +def observe_queued_grant(context: NativeNegativeContext, issuer_pid: int, *, target_pid: int, target_start_token: str, issuer_start_token: str) -> dict[str, object]: + """Prove a full grant reached the stopped receiver before the issuer dies.""" + _require_publication_prefix(context, issuer_pid, "receipt") + with ExitStack() as held: + target = DockerTraceProcess(context) + held.callback(target.close) + issuer_lifetime = os.pidfd_open(issuer_pid, 0) + held.callback(os.close, issuer_lifetime) + issuer_dead = select.poll() + issuer_dead.register(issuer_lifetime, select.POLLIN) + + def check_lifetimes() -> None: + target.check() + if (target.pid, target.start_token) != (target_pid, target_start_token) or issuer_dead.poll(0) or _token(issuer_pid) != issuer_start_token: + raise AssertionError("a queued-grant observer input differs from the original pinned lifetimes") + + check_lifetimes() + before = _descriptor_snapshot(target.pid) + _stopped_receiver(target.pid, target.start_token, target.pidfd, before) + if len(before) > 4096: + raise AssertionError("the actual receiver descriptor inventory exceeds the fixture bound") + expected = _expected_queued_grant(context, issuer_pid, issuer_start_token, target.pid, target.start_token) + configuration = _consumer_configuration(context) + matches = [] + for descriptor, device, inode, _ in before: + if not stat.S_ISSOCK(Path(f"/proc/{target.pid}/fd/{descriptor}").stat().st_mode): + continue + duplicate = _duplicate(target.pid, descriptor, device, inode) + held.callback(os.close, duplicate) + credentials = _local_socket_peer(duplicate) + if credentials is not None and credentials["pid"] == issuer_pid: + if credentials["uid"] != configuration.issuer_uid: + raise AssertionError("the actual queued-grant sender has the wrong Unix owner") + matches.append((descriptor, device, inode, duplicate, credentials)) + if len(matches) != 1: + raise AssertionError("the stopped receiver does not retain one exact authenticated issuer socket") + descriptor, device, inode, duplicate, credentials = matches[0] + encoded, flags_before, flags_after = _peek_queued_packet(duplicate, canonical(expected) + b"\n") + check_lifetimes() + _stopped_receiver(target.pid, target.start_token, target.pidfd, before) + _assert_no_target_admission(context, running_pid=target.pid) + _require_publication_prefix(context, issuer_pid, "receipt") + if _expected_queued_grant(context, issuer_pid, issuer_start_token, target.pid, target.start_token) != expected or _local_socket_peer(duplicate) != credentials: + raise AssertionError("the original grant binding or socket sender changed during kernel observation") + logger.bind(issuer_pid=issuer_pid, target_pid=target.pid, grant_schema=GRANT_SCHEMA, bytes_observed=len(encoded)).info("Observed the complete consumed grant still queued for the stopped actual receiver") + return { + "schema": "actual-recovery-queued-grant-v1", + "target_id": configuration.target_id, + "target_pid": target.pid, + "target_start_token": target.start_token, + "issuer_pid": issuer_pid, + "issuer_start_token": issuer_start_token, + "socket_descriptor": descriptor, + "socket_device": device, + "socket_inode": inode, + "peer_credentials": credentials, + "grant_schema": GRANT_SCHEMA, + "message_sha256": hashlib.sha256(encoded).hexdigest(), + "bytes_observed": len(encoded), + "grant": expected, + "socket_flags_before": flags_before, + "socket_flags_after": flags_after, + "peek_only": True, + "socket_flags_unchanged": True, + "bytes_still_queued": True, + "receiver_stopped": True, + "issuer_alive": True, + } + + @contextmanager def pause_target_before_grant(context: NativeNegativeContext, issuer_pid: int) -> Iterator[dict[str, object]]: """Hold the actual receiver while the original issuer sends, then dies.""" @@ -670,11 +838,11 @@ def _descriptor_snapshot(pid: int) -> tuple[tuple[int, int, int, str], ...]: def _stopped_receiver(pid: int, start_token: str, pidfd: int, descriptors: tuple[tuple[int, int, int, str], ...]) -> dict[str, object]: - """Prove refusal preceded receiver death or any userspace acknowledgment.""" + """Prove the exact live receiver cannot yet observe userspace messages.""" poll = select.poll() poll.register(pidfd, select.POLLIN) if poll.poll(0) or _token(pid) != start_token: - raise AssertionError("the receiver lifetime ended before the original issuer refused") + raise AssertionError("the receiver lifetime ended before the original boundary observation") state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] if state not in {"T", "t"} or _descriptor_snapshot(pid) != descriptors: raise AssertionError("the receipt-free receiver is not still stopped with its exact descriptors") diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 256066ff9..b41adebbf 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -31,7 +31,7 @@ from scripts.recovery_epoch_containment_fixture import CapturedGeneration, capture_generation, enroll_recovery_scope, selected_file, start_watcher, target_from_container from scripts.recovery_epoch_fault_tracer import BOUNDARY_SCENARIOS, PUBLICATIONS, KernelPublicationBarrier, PublicationObservation, parse_boundary from scripts.recovery_epoch_native_negatives import REQUIRED_NEGATIVE_CASES, REQUIRED_REPORT_CASES, NativeNegativeContext, _replacement, assert_unreserved_epoch, run_native_negatives, verify_native_containment_reports -from scripts.recovery_epoch_native_peer import LATE_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, mutate_after_intent, pause_target_before_grant, run_native_peer_cases +from scripts.recovery_epoch_native_peer import LATE_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, mutate_after_intent, observe_queued_grant, pause_target_before_grant, run_native_peer_cases from scripts.test_management_containment_closed_host import InhibitedStockRoutes, WatcherFixture, require_disposable_host from scripts.test_management_containment_enrollment_host import DisposableSystemdRoutes, _private_directory, _write_private @@ -45,7 +45,7 @@ from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, source_identity from auto_coder.management.recovery_containment_model import RecoveryWatcher from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, AUTHORIZATION_ROOT, AUTHORIZATION_SCHEMA, PEER_ROOT, PEER_SOCKET, EpochAuthorization -from auto_coder.management.recovery_epoch_consumer import ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource +from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA, ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource from auto_coder.management.recovery_epoch_state import EPOCH_ROOT, EpochInventory from auto_coder.management.self_update_handoff import runtime_for_receipt from auto_coder.management.version_staging import ManagementPrepareIntent, ManagementReadyReceipt, ManagementRuntimeConfig, ManagementVersionStager @@ -89,6 +89,15 @@ def required_cases(scenario: str) -> frozenset[str]: return (REQUIRED_CASES - {"complete-native-epoch-granted", "old-owner-and-controller-contenders-refused", "issuer-death-ended-current-custody", "consumed-epoch-replay-refused", "reservation-barrier-contender-refused"}) | {"native-" + scenario, "interrupted-epoch-replay-refused"} +def _fixture_directory() -> Path: + """Select a unique private leaf that fits the actual Linux socket paths.""" + root = FIXTURE_ROOT / uuid.uuid4().hex[:16] + sockets = (PEER_SOCKET, *(root / "deployment" / name / "guard" / "fresh-guard.sock" for name in ("A", "B", "C", "D"))) + if any(len(os.fsencode(path)) > 107 for path in sockets): + raise ValueError("the native fixture's original Unix socket pathname exceeds the Linux limit") + return root + + class WorkerFixture(Chain): """Reuse the real worker producer in roots unrelated to management.""" @@ -596,6 +605,19 @@ def validate_result(value: dict[str, object]) -> None: assert value["durable"]["grant_send_completed"] is (scenario == "lost-reply") if scenario == "lost-reply": assert value["durable"]["receiver_paused_before_send"] is True and value["durable"]["issuer_died_before_receiver_resumed"] is True + original = value["durable"]["native_observation"] + queued = value["durable"]["queued_grant"] + assert queued["schema"] == "actual-recovery-queued-grant-v1" and queued["grant_schema"] == GRANT_SCHEMA + assert all(queued.get(name) is True for name in ("peek_only", "socket_flags_unchanged", "bytes_still_queued", "receiver_stopped", "issuer_alive")) + assert queued["bytes_observed"] > 0 and re.fullmatch(r"[0-9a-f]{64}", queued["message_sha256"]) + packet = queued["grant"] + encoded = canonical(packet) + b"\n" + assert len(encoded) == queued["bytes_observed"] and hashlib.sha256(encoded).hexdigest() == queued["message_sha256"] + assert (packet["schema"], packet["target_id"], packet["host_pid"], packet["host_start_token"]) == (queued["grant_schema"], queued["target_id"], original["issuer_pid"], original["issuer_start_token"]) + assert queued["socket_inode"] > 0 and queued["socket_flags_before"] == queued["socket_flags_after"] + assert all(queued[name] == original[name] for name in ("target_pid", "target_start_token", "issuer_pid", "issuer_start_token")) + assert queued["peer_credentials"] == {"pid": original["issuer_pid"], "uid": 0, "gid": 0} + assert original["issuer_death_verified_before_receiver_resumed"] is True def _selected_boundary(scenario: str) -> tuple[str, str]: @@ -623,6 +645,7 @@ def _lost_reply(context: NativeNegativeContext, barrier: KernelPublicationBarrie _wait(lambda: "One consumed recovery epoch grant send completed" in issuer_log.read_text(), "actual grant send while its receiver remains kernel-stopped", timeout=1800, process=issuer) assert "Recovery custody acknowledged" not in consumer_log.read_text(), "the stopped receiver observed the grant" assert _snapshot((EPOCH_ROOT,)) == original_epoch + queued = observe_queued_grant(context, issuer.pid, target_pid=observation["target_pid"], target_start_token=observation["target_start_token"], issuer_start_token=observation["issuer_start_token"]) signal.pidfd_send_signal(descriptor, signal.SIGKILL) assert issuer.wait(timeout=30) == -signal.SIGKILL assert "Recovery custody acknowledged" not in consumer_log.read_text() @@ -635,7 +658,7 @@ def _lost_reply(context: NativeNegativeContext, barrier: KernelPublicationBarrie assert _snapshot((EPOCH_ROOT,)) == original_epoch replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) assert replay.returncode == 2 and _snapshot((EPOCH_ROOT,)) == original_epoch, "a consumed epoch was reissued after its reply was lost" - return {"phase": "receipt", "consumer_grant_acknowledged": False, "grant_send_completed": True, "receiver_paused_before_send": True, "issuer_died_before_receiver_resumed": True, "native_observation": observation, "epoch_sha256": original_epoch} + return {"phase": "receipt", "consumer_grant_acknowledged": False, "grant_send_completed": True, "receiver_paused_before_send": True, "issuer_died_before_receiver_resumed": True, "native_observation": observation, "queued_grant": queued, "epoch_sha256": original_epoch} def _publication_interruption(scenario: str, observed: PublicationObservation, issuer_log: Path, consumer_log: Path, target: PreparedTarget) -> dict[str, object]: @@ -825,6 +848,7 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ if re.fullmatch(r"sha256:[0-9a-f]{64}", os.environ.get(name, "")) is None: raise ValueError("an exact already imported fixture image is required: " + name) authorization = require_disposable_host(authorization_id) + root = _fixture_directory() # No resource is provisioned until the original marker selected this VM. os.umask(0o077) for directory in (FIXTURE_ROOT, EPOCH_ROOT, PEER_ROOT, AUTHORIZATION_ROOT): @@ -833,7 +857,6 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ for parent in (Path("/var/lib/auto-coder"), Path("/etc/auto-coder")): _private_directory(parent) _private_directory(FIXTURE_ROOT) - root = FIXTURE_ROOT / uuid.uuid4().hex root.mkdir(mode=0o700) for directory in (EPOCH_ROOT, PEER_ROOT, AUTHORIZATION_ROOT): _private_directory(directory) diff --git a/src/auto_coder/management/fresh_host_observation.py b/src/auto_coder/management/fresh_host_observation.py index dc0549cc6..51a3996a8 100644 --- a/src/auto_coder/management/fresh_host_observation.py +++ b/src/auto_coder/management/fresh_host_observation.py @@ -169,21 +169,6 @@ def capture_prior_container(container_id: str, image_id: str, *, expected_mounts raise HostObservationUnavailable("unsupported prior container containment") initialized_fresh = False allowed_destinations = {"/bootstrap", "/management-child-state", "/management-oauth", "/management-operations", "/management-policy", "/management-versions", "/run/firebase-admin", "/run/secrets/release_worker_bridge", "/run/secrets/beta_worker_bridge", "/etc/caddy/Caddyfile"} - if not require_owner_lock and not require_deployment_identity and values[15:17] == [["caddy"], ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]]: - # The stock Caddy image declares these data volumes. Its explicit - # bind-only profile must select the actual private roots, not silently - # create Docker volumes or admit executable/configuration overlays. - if isinstance(mounts, list): - data_roots = [item for item in mounts if isinstance(item, dict) and item.get("Destination") in ("/data", "/config")] - if data_roots: - if len(data_roots) != 2 or {item.get("Destination") for item in data_roots} != {"/data", "/config"}: - raise HostObservationUnavailable("Caddy data root bindings are incomplete") - for item in data_roots: - path = Path(item.get("Source", "")) - info = path.lstat() - if item.get("Type") != "bind" or item.get("RW") is not True or path != path.resolve(strict=True) or not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o700: - raise HostObservationUnavailable("Caddy data root binding differs") - allowed_destinations.update(("/data", "/config")) if require_deployment_identity: from .exclusion_host import inherited_support diff --git a/src/auto_coder/management/recovery_containment.py b/src/auto_coder/management/recovery_containment.py index 544e316df..19b1bfcf8 100644 --- a/src/auto_coder/management/recovery_containment.py +++ b/src/auto_coder/management/recovery_containment.py @@ -45,6 +45,7 @@ assert_host_listener_absent, assert_prior_network_unreferenced, assert_prior_processes_absent, + capture_prior_container, ) from .installed_bootstrap_mount_profile import _inspect as _full_inspect from .installed_bootstrap_mount_profile import _mounts @@ -517,7 +518,13 @@ def capture_recovery_original_pair(original: RecoveryOriginal, *, source: str) - mounts = _mounts(full) manager = _capture_prior_kernel(original.management_id, original.management_image_id, engine, values, mounts, require_owner_lock=True) manager = _capture_supervisor_identity(manager, values, mounts, Path(original.bootstrap_config.path)) - proxy = capture_original_caddy_container(original.proxy_id, original.proxy_image_id, expected_mounts=_mounts(proxy_full)) + proxy_mounts = _mounts(proxy_full) + if tuple(destination for _, destination, _ in proxy_mounts) == ("/etc/caddy/Caddyfile",): + # Ordinary cutover and original stock Caddy retain their separate + # capture contracts. Recovery composes them without widening either. + proxy = capture_prior_container(original.proxy_id, original.proxy_image_id, expected_mounts=proxy_mounts) + else: + proxy = capture_original_caddy_container(original.proxy_id, original.proxy_image_id, expected_mounts=proxy_mounts) if _full_inspect(original.management_id) != full or _full_inspect(original.proxy_id) != proxy_full or _engine_id() != engine or manager.net_namespace != proxy.net_namespace: raise RecoveryContainmentUnavailable("recovery original changed during live capture") validate_manager_profile(original, full, source=source) diff --git a/src/auto_coder/management/recovery_containment_profile.py b/src/auto_coder/management/recovery_containment_profile.py index 016330560..7164ddebf 100644 --- a/src/auto_coder/management/recovery_containment_profile.py +++ b/src/auto_coder/management/recovery_containment_profile.py @@ -278,6 +278,9 @@ def validate_proxy_profile(selected: RecoveryOriginal, inspected: dict[str, obje if host.get("NetworkMode") != "container:" + selected.management_id or config.get("Entrypoint") != ["caddy"] or config.get("Cmd") != ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] or config.get("WorkingDir") != "/srv": raise RecoveryContainmentUnavailable("original recovery proxy command or network differs") mounts = bind_mounts(inspected, selected.mounts, expected_uid=expected_uid) + effects = {item.destination: item.writable for item in mounts} + if effects not in ({"/etc/caddy/Caddyfile": False}, {"/etc/caddy/Caddyfile": False, "/data": True, "/config": True}) or len(effects) != len(mounts): + raise RecoveryContainmentUnavailable("original recovery proxy mount effects differ") if _mapped_file("/etc/caddy/Caddyfile", mounts, writable=False) != selected.proxy_config.path: raise RecoveryContainmentUnavailable("original recovery proxy configuration is not actually mounted") for item in mounts: diff --git a/tests/test_management_archived_cutover_container.py b/tests/test_management_archived_cutover_container.py index 5aa3f8323..5c5203bf0 100644 --- a/tests/test_management_archived_cutover_container.py +++ b/tests/test_management_archived_cutover_container.py @@ -190,9 +190,7 @@ def __init__(self, root: Path, *, with_workers: bool = False, original_history: def prepare_caddy_image(self) -> str: """Use the real pinned Caddy binary with the supported bind-only profile.""" - if self.original_genesis: - return _resolve_fixture_base(os.environ.get("AUTOCODER_RECOVERY_CADDY_IMAGE", "caddy@sha256:504a44d7d74bd2bc28a66afc4774912fc21223ca8a67734af1c54ab2f999b489")) - source = _resolve_fixture_base("caddy@sha256:504a44d7d74bd2bc28a66afc4774912fc21223ca8a67734af1c54ab2f999b489") + source = _resolve_fixture_base(os.environ["AUTOCODER_RECOVERY_CADDY_IMAGE"] if self.original_genesis else "caddy@sha256:504a44d7d74bd2bc28a66afc4774912fc21223ca8a67734af1c54ab2f999b489") binary = self.root / "caddy" donor = self.docker("create", "--network", "none", source) target = None @@ -498,10 +496,7 @@ def ensure_proxy(self, generation: Generation) -> None: ) def proxy_mounts(self, generation: Generation) -> tuple[tuple[str, str, bool], ...]: - mounts = ((str(generation.path("Caddyfile")), "/etc/caddy/Caddyfile", False),) - if self.original_genesis: - mounts += ((str(generation.path("caddy-data")), "/data", True), (str(generation.path("caddy-config")), "/config", True)) - return mounts + return ((str(generation.path("Caddyfile")), "/etc/caddy/Caddyfile", False),) @staticmethod def _write_proxy_config(root: Path) -> Path: @@ -514,11 +509,9 @@ def _write_proxy_config(root: Path) -> Path: def _create_recovery_proxy(self, generation: Generation) -> None: """Create the real paired object before original capture, without start.""" self._write_proxy_config(generation.root) - for name in ("caddy-data", "caddy-config"): - generation.path(name).mkdir(mode=0o700) name = "original-recovery-proxy-" + uuid.uuid4().hex[:16] self.names.append(name) - arguments = ["create", "--name", name, "--network", "container:" + generation.container, "--entrypoint", "caddy"] + arguments = ["create", "--name", name, "--workdir", "/srv", "--network", "container:" + generation.container, "--entrypoint", "caddy"] for source, destination, writable in self.proxy_mounts(generation): arguments += ["--mount", f"type=bind,src={source},dst={destination}" + ("" if writable else ",readonly")] generation.proxy = self.docker(*arguments, self.proxy_image, "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile") diff --git a/tests/test_management_original_genesis_capture.py b/tests/test_management_original_genesis_capture.py index e968d2e85..e9b31adca 100644 --- a/tests/test_management_original_genesis_capture.py +++ b/tests/test_management_original_genesis_capture.py @@ -1,4 +1,4 @@ -"""Bounded ordinary genesis predecessor and stock Caddy capture profiles.""" +"""Ordinary genesis predecessors preserve the existing strict Caddy capture.""" from __future__ import annotations @@ -50,23 +50,18 @@ def caddy_profile(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): return identifier, image, values, marker -def test_stock_caddy_explicit_private_data_binds_are_captured(caddy_profile) -> None: +def test_original_genesis_proxy_uses_the_existing_single_config_capture(caddy_profile) -> None: identifier, image, values, marker = caddy_profile + values[11] = values[11][:1] expected = tuple((item["Source"], item["Destination"], item["RW"]) for item in values[11]) assert observation.capture_prior_container(identifier, image, expected_mounts=expected) is marker -@pytest.mark.parametrize("mutation", ["partial", "writable-policy", "arbitrary-command", "unprotected-root"]) -def test_caddy_data_profile_cannot_admit_other_mounts(caddy_profile, mutation: str) -> None: +@pytest.mark.parametrize("extra", ["data", "config", "both"]) +def test_ordinary_predecessor_capture_does_not_admit_stock_data_mounts(caddy_profile, extra: str) -> None: identifier, image, values, _ = caddy_profile - if mutation == "partial": - values[11].pop() - elif mutation == "writable-policy": - values[11][-1]["RW"] = False - elif mutation == "arbitrary-command": - values[15] = ["python"] - else: - Path(values[11][-1]["Source"]).chmod(0o755) + if extra != "both": + values[11] = [item for item in values[11] if item["Destination"] in {"/etc/caddy/Caddyfile", "/" + extra}] expected = tuple((item["Source"], item["Destination"], item["RW"]) for item in values[11]) with pytest.raises(observation.HostObservationUnavailable): observation.capture_prior_container(identifier, image, expected_mounts=expected) diff --git a/tests/test_management_recovery_containment.py b/tests/test_management_recovery_containment.py index b444f6639..d6e2851ca 100644 --- a/tests/test_management_recovery_containment.py +++ b/tests/test_management_recovery_containment.py @@ -6,6 +6,7 @@ import io import json import os +import pwd import stat import subprocess import tarfile @@ -188,7 +189,7 @@ def test_named_descriptors_are_exact_sorted_and_have_no_aliases(): def test_protected_fixture_load_rejects_unknown_fields_duplicates_and_permissions(): - with tempfile.TemporaryDirectory(prefix="recovery-enrollment-component-", dir=Path.home()) as directory: + with tempfile.TemporaryDirectory(prefix="recovery-enrollment-component-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: root = Path(directory) path = root / "enrollment.json" selected = _enrollment() diff --git a/tests/test_management_recovery_containment_resources.py b/tests/test_management_recovery_containment_resources.py index 1a076465b..d2e2cc5cc 100644 --- a/tests/test_management_recovery_containment_resources.py +++ b/tests/test_management_recovery_containment_resources.py @@ -9,6 +9,7 @@ import fcntl import hashlib +import inspect import json import os import pwd @@ -18,8 +19,9 @@ import tempfile from contextlib import contextmanager from dataclasses import dataclass, replace +from functools import wraps from pathlib import Path -from typing import Iterator +from typing import Callable, Iterator, ParamSpec import pytest @@ -29,6 +31,86 @@ pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") +_PARAMETERS = ParamSpec("_PARAMETERS") +_CASE_PARAMETER_SETS: dict[str, tuple[tuple[tuple[str, str], ...], ...]] = { + "test_real_independent_resource_holder_is_observed_without_observer_flock_authority": ((),), + "test_inherited_open_file_description_is_not_independent_issuer_acquisition": tuple((("mode", value),) for value in ("inherited-original", "inherited-controller")), + "test_real_resource_change_at_observed_barrier_refuses_and_closes_observer_references": tuple( + (("change", value),) for value in ("unlock-original", "unlock-controller", "close-original", "close-controller", "close-namespace", "replace-original", "replace-controller", "replace-namespace", "holder-death", "original-pin-closed", "owner-path-replaced") + ), + "test_required_resource_group_cannot_be_omitted": tuple((("field", value),) for value in ("original_owner_fds", "controller_fds", "namespace_fds")), + "test_holder_lifetime_binding_cannot_name_another_process_instance": ((("field", "holder_start_token"), ("value", "1")), (("field", "holder_boot_id"), ("value", "00000000-0000-0000-0000-000000000000"))), +} +_RESOURCE_ASSERTIONS: dict[str, Callable[..., None]] = {} +_CASE_RECEIPT = "RECOVERY_RESOURCE_CASE_COMPLETED " + + +def _selected_resource_case(raw: object) -> tuple[str, dict[str, str]]: + if not isinstance(raw, dict) or set(raw) != {"case", "parameters"}: + raise ValueError("resource component invocation is malformed") + name, parameters = raw["case"], raw["parameters"] + if not isinstance(name, str) or name not in _CASE_PARAMETER_SETS or not isinstance(parameters, dict) or any(not isinstance(key, str) or not isinstance(value, str) for key, value in parameters.items()): + raise ValueError("resource component invocation is not selected") + if tuple(sorted(parameters.items())) not in _CASE_PARAMETER_SETS[name]: + raise ValueError("resource component parameters are not selected") + return name, parameters + + +def _read_host_namespaces() -> None: + for namespace in ("pid", "user", "mnt", "net"): + os.readlink("/proc/1/ns/" + namespace) + + +def _native_resource_case(function: Callable[_PARAMETERS, None]) -> Callable[_PARAMETERS, None]: + """Run the identical real assertions with CI's explicitly available sudo. + + The parent pytest node remains responsible for its result and source receipt. + Local test execution never escalates automatically, and an unavailable host + component is a failure rather than evidence of native VM acceptance. + """ + _RESOURCE_ASSERTIONS[function.__name__] = function + + @wraps(function) + def run(*args: _PARAMETERS.args, **kwargs: _PARAMETERS.kwargs) -> None: + try: + _read_host_namespaces() + except OSError as exc: + if os.environ.get("GITHUB_ACTIONS") != "true": + pytest.fail("Actual host namespace access is unavailable for this component test; run it explicitly with the required host permissions. This is not native VM acceptance: " + str(exc)) + parameters = inspect.signature(function).bind(*args, **kwargs).arguments + name, selected = _selected_resource_case({"case": function.__name__, "parameters": parameters}) + invocation = json.dumps({"case": name, "parameters": selected}, sort_keys=True) + root = Path(__file__).resolve().parents[1] + command = ["sudo", "-n", sys.executable, "-I", "-B", str(Path(__file__).resolve()), "--resource-case", invocation] + try: + completed = subprocess.run(command, cwd=root, capture_output=True, text=True, timeout=60, check=False) + except (OSError, subprocess.TimeoutExpired) as failure: + pytest.fail("Genuine privileged resource component execution is unavailable: " + str(failure)) + assert completed.returncode == 0, "Genuine privileged resource assertions did not complete:\n" + completed.stdout + completed.stderr + receipts = [json.loads(line.removeprefix(_CASE_RECEIPT)) for line in completed.stdout.splitlines() if line.startswith(_CASE_RECEIPT)] + assert len(receipts) == 1, "The selected resource assertions did not produce their completion receipt" + receipt = receipts[0] + assert receipt == {"case": name, "parameters": selected, "uid": 0, "source": str(Path(__file__).resolve())} + return + function(*args, **kwargs) + + return run + + +def _run_resource_case(arguments: list[str]) -> None: + if len(arguments) != 2 or arguments[0] != "--resource-case": + raise ValueError("only an explicitly selected resource component case can run") + name, parameters = _selected_resource_case(json.loads(arguments[1])) + if os.geteuid() != 0: + raise RuntimeError("the privileged resource component child did not obtain root") + expected_source = Path(__file__).resolve().parents[1] / "src/auto_coder/management/recovery_containment_resources.py" + if Path(inspect.getsourcefile(inspect.unwrap(hold_observed_resources)) or "").resolve() != expected_source: + raise RuntimeError("the resource component child imported another checkout") + _read_host_namespaces() + _RESOURCE_ASSERTIONS[name](**parameters) + print(_CASE_RECEIPT + json.dumps({"case": name, "parameters": parameters, "uid": os.geteuid(), "source": str(Path(__file__).resolve())}, sort_keys=True), flush=True) + + _HOLDER = r""" import fcntl,json,os,sys from pathlib import Path @@ -153,6 +235,7 @@ def resource_case(mode: str = "independent") -> Iterator[ResourceCase]: pass +@_native_resource_case def test_real_independent_resource_holder_is_observed_without_observer_flock_authority() -> None: with resource_case() as case: before = set(os.listdir("/proc/self/fd")) @@ -182,6 +265,7 @@ def test_real_independent_resource_holder_is_observed_without_observer_flock_aut @pytest.mark.parametrize("mode", ["inherited-original", "inherited-controller"]) +@_native_resource_case def test_inherited_open_file_description_is_not_independent_issuer_acquisition(mode: str) -> None: with resource_case(mode) as case: before = set(os.listdir("/proc/self/fd")) @@ -192,11 +276,14 @@ def test_inherited_open_file_description_is_not_independent_issuer_acquisition(m @pytest.mark.parametrize("change", ["unlock-original", "unlock-controller", "close-original", "close-controller", "close-namespace", "replace-original", "replace-controller", "replace-namespace", "holder-death", "original-pin-closed", "owner-path-replaced"]) +@_native_resource_case def test_real_resource_change_at_observed_barrier_refuses_and_closes_observer_references(change: str) -> None: with resource_case() as case: before = len(os.listdir("/proc/self/fd")) + entered = injected = False with pytest.raises((ValueError, OSError)): with hold_observed_resources(case.enrollment, case.resources, expected_uid=os.geteuid()): + entered = True if change == "holder-death": case.process.kill() case.process.wait(timeout=5) @@ -209,23 +296,64 @@ def test_real_resource_change_at_observed_barrier_refuses_and_closes_observer_re os.close(descriptor) else: case.command(change) + injected = True + assert entered, "the resource observation failed before the real mutation barrier" + assert injected, "the intended real resource mutation did not complete" expected = before - int(change == "original-pin-closed") assert len(os.listdir("/proc/self/fd")) == expected @pytest.mark.parametrize("field", ["original_owner_fds", "controller_fds", "namespace_fds"]) +@_native_resource_case def test_required_resource_group_cannot_be_omitted(field: str) -> None: with resource_case() as case: - changed = replace(case.resources, **{field: ()}) + if field == "original_owner_fds": + changed = replace(case.resources, original_owner_fds=()) + elif field == "controller_fds": + changed = replace(case.resources, controller_fds=()) + else: + assert field == "namespace_fds" + changed = replace(case.resources, namespace_fds=()) with pytest.raises(RecoveryContainmentUnavailable, match="membership"): with hold_observed_resources(case.enrollment, changed, expected_uid=os.geteuid()): pytest.fail("missing required resource group was accepted") @pytest.mark.parametrize("field,value", [("holder_start_token", "1"), ("holder_boot_id", "00000000-0000-0000-0000-000000000000")]) +@_native_resource_case def test_holder_lifetime_binding_cannot_name_another_process_instance(field: str, value: str) -> None: with resource_case() as case: - changed = replace(case.resources, **{field: value}) + if field == "holder_start_token": + changed = replace(case.resources, holder_start_token=value) + else: + assert field == "holder_boot_id" + changed = replace(case.resources, holder_boot_id=value) with pytest.raises(RecoveryContainmentUnavailable, match="changed or exited"): with hold_observed_resources(case.enrollment, changed, expected_uid=os.geteuid()): pytest.fail("another lifetime was accepted") + + +@pytest.mark.parametrize( + "invocation", + [ + {}, + {"case": "arbitrary", "parameters": {}}, + {"case": "test_real_independent_resource_holder_is_observed_without_observer_flock_authority", "parameters": {"unexpected": "value"}}, + {"case": "test_required_resource_group_cannot_be_omitted", "parameters": {"field": "arbitrary"}}, + {"case": "test_holder_lifetime_binding_cannot_name_another_process_instance", "parameters": {"field": "holder_start_token", "value": "00000000-0000-0000-0000-000000000000"}}, + ], +) +def test_privileged_component_dispatch_rejects_unselected_case_or_parameters(invocation: object) -> None: + with pytest.raises(ValueError, match="resource component"): + _selected_resource_case(invocation) + + +def test_privileged_component_dispatch_covers_every_real_assertion() -> None: + assert set(_RESOURCE_ASSERTIONS) == set(_CASE_PARAMETER_SETS) + for name, selections in _CASE_PARAMETER_SETS.items(): + for parameters in selections: + assert _selected_resource_case({"case": name, "parameters": dict(parameters)}) == (name, dict(parameters)) + + +if __name__ == "__main__": + _run_resource_case(sys.argv[1:]) diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index 3140694e8..4ae2a3a71 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -55,6 +55,26 @@ def refusal(_): assert created == [] and not inputs["output"].exists() +def test_unique_fixture_leaf_fits_the_actual_original_socket_paths() -> None: + first, second = fixture._fixture_directory(), fixture._fixture_directory() + assert first.parent == second.parent == fixture.FIXTURE_ROOT and first != second + assert len(first.name) == 16 and int(first.name, 16) >= 0 + for name in ("A", "B", "C", "D"): + assert len(os.fsencode(first / "deployment" / name / "guard" / "fresh-guard.sock")) <= 107 + assert len(os.fsencode(fixture.PEER_SOCKET)) <= 107 + + +def test_oversized_socket_path_stops_before_any_original_provisioning(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: {}) + monkeypatch.setattr(fixture, "FIXTURE_ROOT", tmp_path / ("long-original-root-" * 8)) + monkeypatch.setattr(fixture, "_private_directory", lambda _: pytest.fail("oversized socket path reached original provisioning")) + monkeypatch.setattr(fixture.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("oversized socket path reached a native effect")) + with pytest.raises(ValueError, match="Unix socket pathname exceeds"): + fixture.run_fixture(**inputs) + assert not inputs["output"].exists() + + def test_mutable_image_tag_stops_before_host_authorization(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: inputs = _inputs(tmp_path, monkeypatch) monkeypatch.setenv("AUTOCODER_FRESH_TEST_BASE_IMAGE", "auto-coder:latest") @@ -275,10 +295,43 @@ def test_native_result_cannot_claim_success_without_preservation(field: str) -> fixture.validate_result(result) +def _lost_reply_durable() -> dict[str, object]: + lifetimes = {"target_pid": 321, "target_start_token": "12345", "issuer_pid": 654, "issuer_start_token": "54321"} + grant = {"schema": fixture.GRANT_SCHEMA, "target_id": "a" * 64, "host_pid": lifetimes["issuer_pid"], "host_start_token": lifetimes["issuer_start_token"]} + encoded = canonical(grant) + b"\n" + return { + "phase": "receipt", + "consumer_grant_acknowledged": False, + "grant_send_completed": True, + "receiver_paused_before_send": True, + "issuer_died_before_receiver_resumed": True, + "native_observation": {**lifetimes, "issuer_death_verified_before_receiver_resumed": True}, + "queued_grant": { + **lifetimes, + "schema": "actual-recovery-queued-grant-v1", + "grant_schema": fixture.GRANT_SCHEMA, + "target_id": grant["target_id"], + "grant": grant, + "message_sha256": hashlib.sha256(encoded).hexdigest(), + "bytes_observed": len(encoded), + "socket_inode": 789, + "socket_flags_before": 2048, + "socket_flags_after": 2048, + "peer_credentials": {"pid": lifetimes["issuer_pid"], "uid": 0, "gid": 0}, + "peek_only": True, + "socket_flags_unchanged": True, + "bytes_still_queued": True, + "receiver_stopped": True, + "issuer_alive": True, + }, + } + + @pytest.mark.parametrize("scenario,phase", [("kill-after-intent", "intent"), ("kill-after-consume", "consumed"), ("lost-reply", "receipt")]) def test_interruption_result_requires_the_actual_boundary_and_no_grant(scenario: str, phase: str) -> None: result = _result() - result.update(scenario=scenario, cases=sorted(fixture.required_cases(scenario)), durable={"phase": phase, "consumer_grant_acknowledged": False, "grant_send_completed": scenario == "lost-reply", "receiver_paused_before_send": True, "issuer_died_before_receiver_resumed": True}) + durable = _lost_reply_durable() if scenario == "lost-reply" else {"phase": phase, "consumer_grant_acknowledged": False, "grant_send_completed": False} + result.update(scenario=scenario, cases=sorted(fixture.required_cases(scenario)), durable=durable) fixture.validate_result(result) result["durable"]["consumer_grant_acknowledged"] = True with pytest.raises(AssertionError): @@ -305,13 +358,49 @@ def test_canonical_acceptance_contains_all_distinct_publication_and_late_faults( @pytest.mark.parametrize("field", ["grant_send_completed", "receiver_paused_before_send", "issuer_died_before_receiver_resumed"]) def test_lost_reply_requires_actual_send_before_issuer_death(field: str) -> None: result = _result() - result.update(scenario="lost-reply", cases=sorted(fixture.required_cases("lost-reply")), durable={"phase": "receipt", "consumer_grant_acknowledged": False, "grant_send_completed": True, "receiver_paused_before_send": True, "issuer_died_before_receiver_resumed": True}) + result.update(scenario="lost-reply", cases=sorted(fixture.required_cases("lost-reply")), durable=_lost_reply_durable()) fixture.validate_result(result) result["durable"][field] = False with pytest.raises(AssertionError): fixture.validate_result(result) +@pytest.mark.parametrize("field", ["peek_only", "socket_flags_unchanged", "bytes_still_queued", "receiver_stopped", "issuer_alive"]) +def test_lost_reply_requires_nonconsuming_current_kernel_observation(field: str) -> None: + result = _result() + result.update(scenario="lost-reply", cases=sorted(fixture.required_cases("lost-reply")), durable=_lost_reply_durable()) + fixture.validate_result(result) + result["durable"]["queued_grant"][field] = False + with pytest.raises(AssertionError): + fixture.validate_result(result) + + +@pytest.mark.parametrize("mutation", ["missing", "foreign-peer", "foreign-lifetime", "changed-flags", "empty-queue", "issuer-still-alive", "changed-packet", "foreign-grant"]) +def test_lost_reply_log_cannot_replace_exact_queued_bytes_and_lifetimes(mutation: str) -> None: + result = _result() + result.update(scenario="lost-reply", cases=sorted(fixture.required_cases("lost-reply")), durable=_lost_reply_durable()) + if mutation == "missing": + result["durable"].pop("queued_grant") + elif mutation == "foreign-peer": + result["durable"]["queued_grant"]["peer_credentials"]["pid"] += 1 + elif mutation == "foreign-lifetime": + result["durable"]["queued_grant"]["target_start_token"] = "99999" + elif mutation == "changed-flags": + result["durable"]["queued_grant"]["socket_flags_after"] += 1 + elif mutation == "empty-queue": + result["durable"]["queued_grant"]["bytes_observed"] = 0 + elif mutation in {"changed-packet", "foreign-grant"}: + queued = result["durable"]["queued_grant"] + queued["grant"]["target_id"] = "b" * 64 + if mutation == "foreign-grant": + encoded = canonical(queued["grant"]) + b"\n" + queued.update(message_sha256=hashlib.sha256(encoded).hexdigest(), bytes_observed=len(encoded)) + else: + result["durable"]["native_observation"]["issuer_death_verified_before_receiver_resumed"] = False + with pytest.raises((AssertionError, KeyError)): + fixture.validate_result(result) + + @pytest.mark.parametrize("case", sorted(fixture.REQUIRED_PEER_CASES)) def test_grant_result_requires_the_actual_peer_evidence_inventory(case: str) -> None: result = _result() diff --git a/tests/test_management_recovery_epoch_native_peer.py b/tests/test_management_recovery_epoch_native_peer.py index 93372ab1a..ad94efd90 100644 --- a/tests/test_management_recovery_epoch_native_peer.py +++ b/tests/test_management_recovery_epoch_native_peer.py @@ -26,7 +26,9 @@ _descriptor_snapshot, _failed_terminal, _Iovec, + _local_socket_peer, _MessageHeader, + _peek_queued_packet, _stopped_receiver, _verify_target_loss, mutate_after_intent, @@ -347,3 +349,67 @@ def test_live_stopped_receiver_oracle_cannot_count_running_or_dead_peer_as_unack signal.pidfd_send_signal(descriptor, signal.SIGKILL) process.communicate(timeout=10) os.close(descriptor) + + +@pytest.mark.parametrize("blocking", [True, False]) +def test_actual_queued_grant_peek_preserves_bytes_and_shared_socket_flags(blocking: bool) -> None: + """Use a real duplicate OFD; native VM separately requires pidfd_getfd.""" + import fcntl + + expected = canonical({"schema": "management-recovery-epoch-grant-v1", "component": "queued-frame"}) + b"\n" + with ExitStack() as held: + sender, receiver = socket.socketpair() + held.enter_context(sender) + held.enter_context(receiver) + receiver.setblocking(blocking) + duplicate = os.dup(receiver.fileno()) + held.callback(os.close, duplicate) + before = fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) + sender.sendall(expected) + encoded, observed_before, observed_after = _peek_queued_packet(duplicate, expected) + assert encoded == expected and observed_before == observed_after == before + assert fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) == before + assert _local_socket_peer(duplicate) == {"pid": os.getpid(), "uid": os.geteuid(), "gid": os.getegid()} + # This is the first consuming read. Both native peeks left the whole + # original frame available to the intended receiver's own descriptor. + assert receiver.recv(len(expected) + 1, socket.MSG_DONTWAIT) == expected + with pytest.raises(BlockingIOError): + receiver.recv(1, socket.MSG_DONTWAIT) + + +@pytest.mark.parametrize("fault", ["empty", "closed", "partial", "foreign", "additional", "oversized"]) +def test_actual_queued_grant_peek_cannot_substitute_expected_records_for_received_bytes(fault: str) -> None: + import fcntl + + expected = canonical({"schema": "management-recovery-epoch-grant-v1", "request_id": "component-request"}) + b"\n" + with ExitStack() as held: + sender, receiver = socket.socketpair() + held.enter_context(sender) + held.enter_context(receiver) + before = fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) + payload = b"" + if fault == "closed": + sender.close() + elif fault == "partial": + payload = expected[:-1] + elif fault == "foreign": + payload = canonical({"schema": "management-recovery-epoch-grant-v1", "request_id": "foreign-request"}) + b"\n" + elif fault == "additional": + payload = expected + b"unexpected-second-frame\n" + elif fault == "oversized": + payload = b"x" * (MAX_MESSAGE_BYTES + 1) + if payload: + sender.sendall(payload) + with pytest.raises(AssertionError, match="queued grant|consumed grant"): + _peek_queued_packet(receiver.fileno(), expected) + assert fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) == before + if payload: + assert receiver.recv(MAX_MESSAGE_BYTES + 2, socket.MSG_DONTWAIT) == payload + + +def test_actual_socket_peer_reader_does_not_accept_datagram_transport() -> None: + with ExitStack() as held: + sender, receiver = socket.socketpair(type=socket.SOCK_DGRAM) + held.enter_context(sender) + held.enter_context(receiver) + assert _local_socket_peer(receiver.fileno()) is None diff --git a/tests/test_management_recovery_proxy_profiles.py b/tests/test_management_recovery_proxy_profiles.py new file mode 100644 index 000000000..ba15fcc44 --- /dev/null +++ b/tests/test_management_recovery_proxy_profiles.py @@ -0,0 +1,76 @@ +"""Real protected mount checks for recovery's two explicit proxy profiles. + +The typed selector and Docker description are component inputs, not evidence +of a real running deployment. Full native capture remains a separate VM gate. +""" + +from __future__ import annotations + +import hashlib +import os +import pwd +import tempfile +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path + +import pytest + +from auto_coder.management.containment_route_manifest import DeploymentRootBinding +from auto_coder.management.recovery_containment_model import RecoveryContainmentUnavailable, RecoveryFile +from auto_coder.management.recovery_containment_profile import digest, validate_proxy_profile +from tests.test_management_recovery_containment import _enrollment + + +@contextmanager +def proxy_case(destinations: tuple[tuple[str, bool], ...]): + with tempfile.TemporaryDirectory(prefix="ac-recovery-proxy-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: + root = Path(directory) + config = root / "Caddyfile" + config.write_bytes(b"{\n admin off\n auto_https off\n}\n") + config.chmod(0o600) + selected = _enrollment().originals[0] + bindings = [] + mounts = [] + for index, (destination, writable) in enumerate(destinations): + path = config if destination == "/etc/caddy/Caddyfile" else root / str(index) + if path != config: + path.mkdir(mode=0o700) + info = path.stat() + bindings.append(DeploymentRootBinding(selected.proxy_id, destination, str(path), writable, info.st_dev, info.st_ino, info.st_mode)) + mounts.append({"Type": "bind", "Source": str(path), "Destination": destination, "RW": writable}) + info = config.stat() + selected = replace(selected, proxy_config=RecoveryFile(str(config), hashlib.sha256(config.read_bytes()).hexdigest(), info.st_dev, info.st_ino), mounts=tuple(bindings)) + inspection = { + "Id": selected.proxy_id, + "Image": selected.proxy_image_id, + "HostConfig": {"Privileged": False, "PidMode": "", "CapAdd": None, "Devices": [], "SecurityOpt": None, "NetworkMode": "container:" + selected.management_id}, + "Config": {"User": "", "WorkingDir": "/srv", "Entrypoint": ["caddy"], "Cmd": ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]}, + "Mounts": mounts, + } + yield selected, inspection + + +@pytest.mark.parametrize("destinations", [(("/etc/caddy/Caddyfile", False),), (("/etc/caddy/Caddyfile", False), ("/data", True), ("/config", True))]) +def test_recovery_proxy_accepts_exact_ordinary_or_original_storage_effects(destinations): + with proxy_case(destinations) as (selected, inspection): + assert validate_proxy_profile(selected, inspection, expected_uid=os.geteuid()) == digest([inspection]) + + +@pytest.mark.parametrize( + "destinations", + [ + (("/etc/caddy/Caddyfile", False), ("/data", True)), + (("/etc/caddy/Caddyfile", False), ("/config", True)), + (("/etc/caddy/Caddyfile", False), ("/data", False), ("/config", True)), + (("/etc/caddy/Caddyfile", False), ("/data", True), ("/config", False)), + (("/etc/caddy/Caddyfile", True),), + (("/etc/caddy/Caddyfile", False), ("/unselected", True)), + (("/etc/caddy/Caddyfile", False), ("/unselected", False)), + (("/etc/caddy/Caddyfile", False), ("/data", True), ("/config", True), ("/extra", False)), + ], +) +def test_recovery_proxy_refuses_partial_or_additional_effects(destinations): + with proxy_case(destinations) as (selected, inspection): + with pytest.raises(RecoveryContainmentUnavailable, match="mount effects"): + validate_proxy_profile(selected, inspection, expected_uid=os.geteuid()) From 9d6df50397b9a5dd1f3ad20c98c1ed90adef8fdd Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 02:10:06 +0900 Subject: [PATCH 08/24] test(management): verify the consumed epoch before dispatch --- .../workflows/management-recovery-epoch.yml | 74 ++++- .../management-recovery-epoch.md | 31 ++- scripts/recovery_epoch_mutation_control.py | 253 ++++++++++++++++++ scripts/recovery_epoch_native_peer.py | 150 +++++++++-- scripts/recovery_vm/README.md | 34 ++- scripts/recovery_vm/guest.sh | 41 ++- scripts/recovery_vm/prepare.sh | 13 +- scripts/recovery_vm/runner.sh | 27 +- .../test_management_recovery_epoch_host.py | 127 ++++++++- .../recovery_epoch_authorization.py | 11 +- ...management_recovery_epoch_authorization.py | 55 +++- tests/test_management_recovery_epoch_host.py | 75 +++++- ...agement_recovery_epoch_mutation_control.py | 186 +++++++++++++ ...t_management_recovery_epoch_native_peer.py | 29 +- 14 files changed, 1056 insertions(+), 50 deletions(-) create mode 100644 scripts/recovery_epoch_mutation_control.py create mode 100644 tests/test_management_recovery_epoch_mutation_control.py diff --git a/.github/workflows/management-recovery-epoch.yml b/.github/workflows/management-recovery-epoch.yml index 319d2305f..c2d21bd0d 100644 --- a/.github/workflows/management-recovery-epoch.yml +++ b/.github/workflows/management-recovery-epoch.yml @@ -121,9 +121,79 @@ jobs: path: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.scenario }}/output/ if-no-files-found: error retention-days: 7 + verifier-omission-control: + name: Detect omitted post-consumption verification in isolated VM + needs: acceptance-plan + runs-on: ubuntu-24.04 + timeout-minutes: 150 + env: + UV_LOCKED: "1" + PYTHONDONTWRITEBYTECODE: "1" + RECOVERY_FIXTURE: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-verifier-omission-control + APPROVED_AUTHORIZATION_ID: ci-recovery-${{ github.run_id }}-${{ github.run_attempt }}-verifier-omission-control + RECOVERY_SCENARIO: post-receipt-ready-mutation + steps: + - name: Checkout exact reviewed source + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.7.22" + - name: Install locked test dependencies + run: uv sync --locked --dev --extra test + - name: Inspect the actual guest acceleration device + run: | + set -euo pipefail + if test -c /dev/kvm; then + stat --format='KVM device: %n mode=%a uid=%u gid=%g' /dev/kvm + else + echo 'No KVM device: the fresh QEMU guest will use bounded software CPU emulation.' + fi + - name: Build and verify exact installed management image + run: uv run python scripts/prepare_management_test_image.py + - name: Prepare the approved fresh recovery VM + env: + FIXTURE_READ_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + umask 077 + token_file="$RUNNER_TEMP/recovery-source-read-token" + trap 'rm -f "$token_file"' EXIT + printf '%s' "$FIXTURE_READ_TOKEN" > "$token_file" + docker pull caddy:2 + manager_id="$(docker image inspect --format '{{.Id}}' "$AUTOCODER_FRESH_TEST_BASE_IMAGE")" + caddy_id="$(docker image inspect --format '{{.Id}}' caddy:2)" + bash scripts/recovery_vm/prepare.sh "$RECOVERY_FIXTURE" "$APPROVED_AUTHORIZATION_ID" "$manager_id" "$caddy_id" "$token_file" "$RECOVERY_SCENARIO" omit-post-consumption-verifier + - name: Build bounded QEMU runner + run: docker compose --env-file "$RECOVERY_FIXTURE/compose.env" build + - name: Detect the exact omitted verifier with the unchanged native oracle + timeout-minutes: 125 + run: | + set -euo pipefail + docker compose --env-file "$RECOVERY_FIXTURE/compose.env" run --rm fixture + - name: Remove the ephemeral read credential + if: always() + run: | + rm -f "$RUNNER_TEMP/recovery-source-read-token" + rm -f "$RECOVERY_FIXTURE/input/source-token" + - name: Retain the distinct mutation-control evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: management-recovery-epoch-${{ github.run_id }}-${{ github.run_attempt }}-verifier-omission-control + path: /var/tmp/ac-recovery-vm-${{ github.run_id }}-${{ github.run_attempt }}-verifier-omission-control/output/ + if-no-files-found: error + retention-days: 7 acceptance: name: Management Recovery Epoch Acceptance - needs: [acceptance-plan, recovery-epoch] + needs: [acceptance-plan, recovery-epoch, verifier-omission-control] if: always() runs-on: ubuntu-24.04 timeout-minutes: 2 @@ -132,6 +202,8 @@ jobs: env: PLAN_RESULT: ${{ needs.acceptance-plan.result }} ACCEPTANCE_RESULT: ${{ needs.recovery-epoch.result }} + MUTATION_CONTROL_RESULT: ${{ needs.verifier-omission-control.result }} run: | test "$PLAN_RESULT" = success test "$ACCEPTANCE_RESULT" = success + test "$MUTATION_CONTROL_RESULT" = success diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md index e185bd7a3..b4aec7c4e 100644 --- a/docs/client-features/management-recovery-epoch.md +++ b/docs/client-features/management-recovery-epoch.md @@ -122,6 +122,14 @@ runtime arguments/environment/source identities must agree with `runtime_for_receipt`. The READY preparation directory must equal the approved preparation ID. Unsupported executable environment hooks refuse. +The installer pin retains the complete `uv --version` text after its `uv ` +prefix, including any build descriptor. Its grammar is a numeric three-part +version with one optional parenthesized ASCII build descriptor, bounded to 128 +characters in total. The stager still requires exact equality with the selected +installer's actual output; a matching numeric version alone cannot authorize a +different build. The native fixture verifies its separately selected numeric +version before passing that complete observed pin into authorization and READY. + Both the complete interpreter distribution and READY environment must be inside the exact same-path read-only version-root mount. In particular, a host-only venv interpreter symlink cannot satisfy target verification. The authorization's @@ -215,7 +223,7 @@ reproduction procedure are documented in `scripts/recovery_vm/README.md`. | AS-B02 | Install the ordinary service/timer/watcher/alternate-root fixture routes, bind them in approved enrollment and verify their actual inhibition. Exercise unknown/omitted/startable controllers, retained namespace listeners, an escaped nonleader task and independently measured exit. Repeat the real containment verifier in a separate process and preserve unrelated worker claims and effect counters. | | AS-B03 | Use the real host and Docker bootstrap processes, kernel peer credentials/pidfds and independent SH/target EX acquisition. Refuse busy/replaced/missing fences, a foreign acquiring process, queued rights without acknowledgment, received rights before acknowledgment, and a changed target lifetime. Later successful issuance must retain the complete observed custody. | | AS-B04 | Reject actual CLI/environment root and inventory overrides. Stop the real issuer at its fsynced reservation head while a contender runs, then test conflicting target/preparation binding. In separate fresh VMs, SIGKILL the original process before and after every record/head publication and inspect the exact retained prefix and refused replay. | -| AS-B05 | After the original intent head, change exact configuration, READY, installed source, original evidence, watcher/controller state, fence acquisition or target lifetime. Require refusal before consumption. Separately stop the receiver, require the real grant send to succeed, prove issuer death through its pidfd, then resume the receiver and require unavailable current custody with no replay. | +| AS-B05 | After the original intent head, change exact configuration, READY, installed source, original evidence, watcher/controller state, fence acquisition or target lifetime. Require refusal before consumption. In a distinct fresh VM, change exact READY after the fsynced receipt head and resume the same issuer; require actual refusal and an empty authenticated receiver queue while the receiver is stopped. Preserve consumed records, the injected mutation and workers, and refuse replay. Separately require a real queued grant before issuer death, then resume the receiver and require unavailable current custody with no replay. | Publication and protocol fault helpers only observe real syscall entry/exit and kernel descriptors. They do not replace system-call results, inject verifier @@ -227,3 +235,24 @@ worker ownership and effect requests, and the absence of any replacement child or application admission. The current PR's custody endpoint and a separately qualified installed READY are exercised independently; these checks do not claim that the separate #2825 management-child launch/readiness contract is complete. + +The separate `omit-post-consumption-verifier` mutation-control job proves that +the post-receipt scenario detects omission of the second final verification. +Before creating any original fixture, it copies the project into a fresh private +directory and removes exactly that one standalone call. An AST comparison +retains the first verifier and every other source expression; the original and +copied file digests are retained. The copied project is installed and supplies +every actual host process and fixture image, so source, history and enrollment +identities continue to agree without replacing an authority provider. + +The control runs the same post-receipt READY mutation and the same no-grant +assertion. Its distinct `mutation-detected` result requires the named +`post-receipt-grant-forbidden` failure and a complete canonical grant observed +with `pidfd_getfd` and nonconsuming `MSG_PEEK` on the stopped receiver's exact +authenticated socket. A timeout, process kill, unrelated exception, unavailable +kernel operation or log message cannot satisfy that result. Only after recording +that incorrect dispatch does cleanup end the mutant issuer; the receiver must +still reject current custody with no child/admission, and the consumed epoch, +deliberate READY mutation and unrelated workers must survive refused replay. +This control does not count as production acceptance, and it never changes the +original source checkout or resets an epoch for another attempt. diff --git a/scripts/recovery_epoch_mutation_control.py b/scripts/recovery_epoch_mutation_control.py new file mode 100644 index 000000000..6a16a55b2 --- /dev/null +++ b/scripts/recovery_epoch_mutation_control.py @@ -0,0 +1,253 @@ +"""One explicit native mutation control, separate from production acceptance. + +The control starts from a private project copy before any original fixture is +created. It removes only the post-consumption final verifier, so the real +package identity, Docker images, history and enrollment all agree on that +intentionally incorrect copy. No production source or authority provider is +patched at runtime. +""" + +from __future__ import annotations + +import argparse +import ast +import copy +import hashlib +import json +import os +import re +import shutil +import sys +from pathlib import Path + +from loguru import logger + +CONTROL = "omit-post-consumption-verifier" +CONTROL_SCENARIO = "post-receipt-ready-mutation" +CONTROL_SCHEMA = "auto-coder-native-recovery-mutation-control-result-v1" +SOURCE_SCHEMA = "auto-coder-recovery-mutation-source-v1" +ORACLE = "post-receipt-grant-forbidden" +ISSUER_SOURCE = Path("src/auto_coder/management/recovery_epoch.py") +_IGNORED = frozenset({".git", ".venv", ".agent-tmp", "__pycache__", ".mypy_cache", ".pytest_cache", ".ruff_cache"}) + + +def _canonical(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + +def omit_post_consumption_verifier(source: bytes) -> tuple[bytes, int]: + """Delete exactly the second standalone call, proving the full AST delta.""" + original = ast.parse(source) + definitions = [node for node in original.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "issue_epoch_once"] + if len(definitions) != 1: + raise ValueError("the actual source has no unique issue_epoch_once function") + function = definitions[0] + calls = sorted((node for node in ast.walk(function) if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id == "final_verification"), key=lambda node: node.lineno) + if len(calls) != 2 or any(node.value.args or node.value.keywords for node in calls): + raise ValueError("the actual issuer does not contain exactly two standalone final verifier calls") + first, removed = calls + parents = [node for node in ast.walk(function) if isinstance(getattr(node, "body", None), list) and removed in node.body] + if len(parents) != 1: + raise ValueError("the second final verifier has no unique original statement list") + statements = parents[0].body + index = statements.index(removed) + previous = statements[index - 1] if index else None + if not ( + isinstance(previous, ast.Assign) + and len(previous.targets) == 1 + and isinstance(previous.targets[0], ast.Name) + and previous.targets[0].id == "receipt" + and isinstance(previous.value, ast.Call) + and isinstance(previous.value.func, ast.Attribute) + and isinstance(previous.value.func.value, ast.Name) + and (previous.value.func.value.id, previous.value.func.attr) == ("state", "consume") + and len(previous.value.args) == 1 + and isinstance(previous.value.args[0], ast.Name) + and previous.value.args[0].id == "binding" + and not previous.value.keywords + and first.lineno < previous.lineno < removed.lineno + ): + raise ValueError("the selected second verifier does not directly follow actual durable consumption") + lines = source.splitlines(keepends=True) + if removed.end_lineno != removed.lineno or lines[removed.lineno - 1].strip() != b"final_verification()": + raise ValueError("the second verifier is not one exact removable source statement") + mutant = b"".join(lines[: removed.lineno - 1] + lines[removed.lineno :]) + expected = copy.deepcopy(original) + expected_function = next(node for node in expected.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "issue_epoch_once") + expected_removed = next(node for node in ast.walk(expected_function) if isinstance(node, ast.Expr) and node.lineno == removed.lineno) + expected_parent = next(node for node in ast.walk(expected_function) if isinstance(getattr(node, "body", None), list) and expected_removed in node.body) + expected_parent.body.remove(expected_removed) + if ast.dump(ast.parse(mutant), include_attributes=False) != ast.dump(expected, include_attributes=False): + raise ValueError("the native mutation changed more than the second final verifier") + return mutant, removed.lineno + + +def _files(root: Path) -> dict[str, str]: + result = {} + for directory, names, files in os.walk(root): + names[:] = sorted(name for name in names if name not in _IGNORED) + if any((Path(directory) / name).is_symlink() for name in names): + raise ValueError("the mutation source contains a linked directory") + for name in sorted(files): + path = Path(directory) / name + if path.is_symlink() or not path.is_file(): + raise ValueError("the mutation source contains a nonregular file: " + str(path)) + result[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + return result + + +def prepare(source: Path, destination: Path, evidence: Path) -> dict[str, object]: + """Create one distinct source copy; never overwrite or modify the original.""" + source = source.resolve(strict=True) + if not source.is_dir() or not destination.is_absolute() or destination.exists() or destination.is_symlink() or destination.is_relative_to(source): + raise ValueError("a new separate absolute mutation-copy directory is required") + if not evidence.is_absolute() or evidence.exists() or evidence.is_symlink() or evidence.is_relative_to(source) or evidence.is_relative_to(destination): + raise ValueError("new mutation evidence must be outside both source trees") + original_files = _files(source) + original = (source / ISSUER_SOURCE).read_bytes() + mutant, line = omit_post_consumption_verifier(original) + shutil.copytree(source, destination, ignore=shutil.ignore_patterns(*_IGNORED), copy_function=shutil.copy2) + selected = destination / ISSUER_SOURCE + with selected.open("wb") as stream: + stream.write(mutant) + stream.flush() + os.fsync(stream.fileno()) + copied_files = _files(destination) + changed = [path for path in original_files.keys() | copied_files.keys() if original_files.get(path) != copied_files.get(path)] + if changed != [ISSUER_SOURCE.as_posix()] or _files(source) != original_files: + raise AssertionError("the copied project differs outside the single approved verifier statement") + result = { + "schema": SOURCE_SCHEMA, + "control": CONTROL, + "scenario": CONTROL_SCENARIO, + "source_root": str(source), + "control_root": str(destination), + "changed_file": ISSUER_SOURCE.as_posix(), + "removed_call_lineno": line, + "original_source_sha256": hashlib.sha256(original).hexdigest(), + "mutant_source_sha256": hashlib.sha256(mutant).hexdigest(), + "original_project_sha256": hashlib.sha256(_canonical(original_files)).hexdigest(), + "mutant_project_sha256": hashlib.sha256(_canonical(copied_files)).hexdigest(), + "package_files": {path.removeprefix("src/auto_coder/"): digest for path, digest in copied_files.items() if path.startswith("src/auto_coder/") and path.endswith(".py")}, + "fixture_files": {path: digest for path, digest in copied_files.items() if path.startswith(("scripts/", "tests/")) and path.endswith(".py")}, + "only_second_verifier_removed": True, + "first_verifier_retained": True, + } + descriptor = os.open(evidence, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(_canonical(result)) + stream.flush() + os.fsync(stream.fileno()) + logger.info("Prepared the isolated source mutation at original line {}: {}", line, selected) + return result + + +def verify_source_evidence(path: Path, *, executing: bool = False) -> tuple[dict[str, object], str]: + encoded = path.read_bytes() + value = json.loads(encoded) + if _canonical(value) != encoded or value.get("schema") != SOURCE_SCHEMA or value.get("control") != CONTROL or value.get("scenario") != CONTROL_SCENARIO: + raise ValueError("the mutation source evidence is not the exact supported canonical record") + original = Path(value["source_root"]) / ISSUER_SOURCE + changed = Path(value["control_root"]) / ISSUER_SOURCE + mutant, line = omit_post_consumption_verifier(original.read_bytes()) + if changed.read_bytes() != mutant or value["removed_call_lineno"] != line or value["original_source_sha256"] != hashlib.sha256(original.read_bytes()).hexdigest() or value["mutant_source_sha256"] != hashlib.sha256(mutant).hexdigest(): + raise AssertionError("the retained original or exact single-call mutation changed") + if value.get("only_second_verifier_removed") is not True or value.get("first_verifier_retained") is not True: + raise AssertionError("the mutation omitted its explicit single-verifier proof") + if executing: + from auto_coder.management import recovery_epoch + + package = Path(recovery_epoch.__file__).resolve(strict=True).parents[1] + actual = {item.relative_to(package).as_posix(): hashlib.sha256(item.read_bytes()).hexdigest() for item in sorted(package.rglob("*.py")) if not item.is_symlink()} + if actual != value["package_files"]: + raise AssertionError("the executing issuer package is not the complete original mutation copy") + if Path(__file__).resolve(strict=True).parents[1] != Path(value["control_root"]): + raise AssertionError("the actual fixture scripts are not running from the private copied project") + if any(hashlib.sha256((Path(value["control_root"]) / name).read_bytes()).hexdigest() != digest for name, digest in value["fixture_files"].items()): + raise AssertionError("the copied native fixture scripts changed after the single-call mutation") + return value, hashlib.sha256(encoded).hexdigest() + + +def _validate_queued_grant(queued: dict[str, object]) -> None: + if queued.get("schema") != "actual-recovery-queued-grant-v1" or queued.get("grant_schema") != "management-recovery-epoch-grant-v1": + raise AssertionError("the mutation did not observe the actual canonical recovery grant") + grant = queued["grant"] + assert set(grant) == {"schema", "granted", "request_id", "authorization_id", "target_id", "challenge", "receipt_nonce", "inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256", "host_boot_id", "host_pid", "host_start_token"} + assert grant["schema"] == queued["grant_schema"] and grant["granted"] == "recovery-epoch-consumed" + assert all(re.fullmatch(r"[0-9a-f]{64}", grant[name]) for name in ("challenge", "receipt_nonce", "inventory_sha256", "binding_sha256", "consumed_sha256", "receipt_sha256")) + encoded = _canonical(grant) + b"\n" + assert queued["bytes_observed"] == len(encoded) and queued["message_sha256"] == hashlib.sha256(encoded).hexdigest() + assert all(queued.get(name) is True for name in ("peek_only", "socket_flags_unchanged", "bytes_still_queued", "receiver_stopped", "issuer_alive")) + assert queued["socket_flags_before"] == queued["socket_flags_after"] and queued["socket_inode"] > 0 + assert queued["peer_credentials"] == {"pid": queued["issuer_pid"], "uid": 0, "gid": 0} + assert (queued["grant"]["target_id"], queued["grant"]["host_pid"], queued["grant"]["host_start_token"]) == (queued["target_id"], queued["issuer_pid"], queued["issuer_start_token"]) + + +class PostReceiptGrantDispatched(AssertionError): + """The same native oracle observed the precise incorrect dispatched frame.""" + + +def assert_post_receipt_refusal(observation: dict[str, object]) -> None: + assert observation["schema"] == "actual-recovery-post-receipt-dispatch-v1" + assert observation["receiver_stopped"] is True and observation["queue_empty_before_resume"] is True and observation["socket_flags_unchanged"] is True + queued = observation["queued_grant"] + if queued is not None: + _validate_queued_grant(queued) + assert observation["queue_bytes_observed"] == queued["bytes_observed"] + assert all(observation[name] == queued[name] for name in ("issuer_pid", "issuer_start_token", "target_pid", "target_start_token", "peer_credentials")) + raise PostReceiptGrantDispatched(ORACLE) + assert observation["queue_bytes_observed"] == 0 and observation["issuer_returncode"] == 2 + + +def validate_control_result(value: dict[str, object], evidence: Path) -> None: + source, digest = verify_source_evidence(evidence) + assert value["schema"] == CONTROL_SCHEMA and value["status"] == "mutation-detected" and value["control"] == CONTROL and value["scenario"] == CONTROL_SCENARIO + assert value["source_evidence_sha256"] == digest and value["source_mutation"] == source + assert value["no_child_started"] is True and value["preserved_originals"] is True and value["preserved_workers"] is True + assert value["original_outcomes"]["D"] == "never-created" and value["original_outcomes"]["C"] in {"launched", "unknown"} + oracle = value["oracle"] + assert oracle["name"] == ORACLE and oracle["detected"] is True + _validate_queued_grant(oracle["queued_grant"]) + durable = value["durable"] + assert durable["phase"] == "receipt" and durable["consumer_grant_acknowledged"] is False and durable["grant_send_completed"] is True + assert durable["mutation"]["scenario"] == CONTROL_SCENARIO and durable["mutation"]["boundary"] == "after-receipt-head" + assert durable["dispatch"]["queued_grant"] == oracle["queued_grant"] + assert durable["dispatch"]["issuer_death_before_receiver_resumed"] is True and durable["replay_returncode"] == 2 + assert durable["retained_publications"] == ["reserved.json", "reserved-head.json", "intent.json", "intent-head.json", "consumed.json", "consumed-head.json", "receipt.json", "receipt-head.json"] + assert durable["epoch_sha256_before_resume"] == durable["epoch_sha256_after_refusal"] == durable["epoch_sha256_after_replay"] + assert durable["originals_after_injection_sha256"] == value["preserved_originals_sha256"] + assert value["preservation_baseline"] == durable["mutation"]["preservation_baseline"] == "after-deliberate-fixture-mutation" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + create = commands.add_parser("prepare") + create.add_argument("--source", type=Path, required=True) + create.add_argument("--destination", type=Path, required=True) + create.add_argument("--evidence", type=Path, required=True) + run = commands.add_parser("run") + run.add_argument("--evidence", type=Path, required=True) + run.add_argument("arguments", nargs=argparse.REMAINDER) + validate = commands.add_parser("validate") + validate.add_argument("--evidence", type=Path, required=True) + validate.add_argument("--result", type=Path, required=True) + args = parser.parse_args(argv) + logger.remove() + logger.add(sys.stdout, colorize=True) + logger.add(str(args.evidence) + ".log", rotation="10 MB") + if args.command == "prepare": + prepare(args.source, args.destination, args.evidence) + elif args.command == "validate": + validate_control_result(json.loads(args.result.read_bytes()), args.evidence) + else: + verify_source_evidence(args.evidence, executing=True) + from scripts.test_management_recovery_epoch_host import main as run_fixture + + arguments = args.arguments[1:] if args.arguments[:1] == ["--"] else args.arguments + return run_fixture([*arguments, "--mutation-control-manifest", str(args.evidence)]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/recovery_epoch_native_peer.py b/scripts/recovery_epoch_native_peer.py index 323d9f79c..8dda9f0d9 100644 --- a/scripts/recovery_epoch_native_peer.py +++ b/scripts/recovery_epoch_native_peer.py @@ -10,6 +10,7 @@ import array import ctypes +import errno import fcntl import hashlib import json @@ -56,6 +57,8 @@ "late-controller-fence", "late-target-loss", ) +POST_RECEIPT_MUTATION_SCENARIOS = ("post-receipt-ready-mutation",) +MUTATION_SCENARIOS = (*LATE_MUTATION_SCENARIOS, *POST_RECEIPT_MUTATION_SCENARIOS) _SYS_SENDTO = 44 _SYS_SENDMSG = 46 _SYS_RECVMSG = 47 @@ -371,7 +374,7 @@ def _mutation_file(context: NativeNegativeContext, scenario: str) -> Path: authorization = EpochAuthorization.parse(json.loads(AUTHORIZATION_FILE.read_bytes())) if authorization.target_id != target.management_id: raise AssertionError("the actual fixed authorization selects a different target") - if scenario == "late-ready-receipt": + if scenario in {"late-ready-receipt", "post-receipt-ready-mutation"}: return Path(authorization.ready_receipt.path) if scenario == "late-installed-source": receipt = json.loads(Path(authorization.ready_receipt.path).read_bytes()) @@ -380,12 +383,13 @@ def _mutation_file(context: NativeNegativeContext, scenario: str) -> Path: @contextmanager -def mutate_after_intent(context: NativeNegativeContext, scenario: str, issuer_pid: int) -> Iterator[dict[str, object]]: +def mutate_after_publication(context: NativeNegativeContext, scenario: str, issuer_pid: int) -> Iterator[dict[str, object]]: """Retain the deliberate fault through refusal and replay; never repair it.""" - if scenario not in LATE_MUTATION_SCENARIOS: + if scenario not in MUTATION_SCENARIOS: raise ValueError("a reviewed late mutation scenario is required") - _require_publication_prefix(context, issuer_pid, "intent") - observation: dict[str, object] = {"scenario": scenario, "issuer_pid": issuer_pid, "issuer_start_token": _token(issuer_pid), "boundary": "after-intent-head", "preservation_baseline": "after-deliberate-fixture-mutation"} + phase = "receipt" if scenario in POST_RECEIPT_MUTATION_SCENARIOS else "intent" + _require_publication_prefix(context, issuer_pid, phase) + observation: dict[str, object] = {"scenario": scenario, "issuer_pid": issuer_pid, "issuer_start_token": _token(issuer_pid), "boundary": "after-" + phase + "-head", "preservation_baseline": "after-deliberate-fixture-mutation"} with ExitStack() as held: if scenario == "late-target-loss": observation["target_fault"] = held.enter_context(_remove_target(context)) @@ -433,7 +437,7 @@ def mutate_after_intent(context: NativeNegativeContext, scenario: str, issuer_pi if before == after or before[:2] != after[:2]: raise AssertionError("the selected native original bytes did not change in place") observation["file_fault"] = {"path": str(path), "before": before, "after": after} - logger.bind(native_peer_fault=scenario, issuer_pid=issuer_pid).info("Injected the explicit native fault after the original intent head") + logger.bind(native_peer_fault=scenario, issuer_pid=issuer_pid).info("Injected the explicit native fault after the original {} head", phase) yield observation if "file_fault" in observation and _projection(path) != after: raise AssertionError("production refusal or replay changed the deliberately injected file") @@ -497,6 +501,46 @@ def peek() -> bytes: return encoded, before, after +def _peek_pending_frame(descriptor: int) -> bytes: + """Observe even an empty real stream queue without changing its shared OFD.""" + libc = ctypes.CDLL(None, use_errno=True) + libc.recv.argtypes = (ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int) + libc.recv.restype = ctypes.c_ssize_t + before = fcntl.fcntl(descriptor, fcntl.F_GETFL) + buffer = ctypes.create_string_buffer(MAX_MESSAGE_BYTES + 1) + result = libc.recv(descriptor, buffer, len(buffer), socket.MSG_PEEK | socket.MSG_DONTWAIT) + error = ctypes.get_errno() + if fcntl.fcntl(descriptor, fcntl.F_GETFL) != before: + raise AssertionError("pending grant observation changed the receiver's shared status flags") + if result < 0: + if error in {errno.EAGAIN, errno.EWOULDBLOCK}: + return b"" + raise OSError(error, os.strerror(error)) + if result > MAX_MESSAGE_BYTES: + raise AssertionError("the actual pending receiver frame exceeds the protocol bound") + return buffer.raw[:result] + + +def _issuer_socket(target: DockerTraceProcess, issuer_pid: int, issuer_uid: int, held: ExitStack) -> tuple[int, int, int, int, dict[str, int]]: + before = _descriptor_snapshot(target.pid) + if len(before) > 4096: + raise AssertionError("the actual receiver descriptor inventory exceeds the fixture bound") + matches = [] + for descriptor, device, inode, _ in before: + if not stat.S_ISSOCK(Path(f"/proc/{target.pid}/fd/{descriptor}").stat().st_mode): + continue + duplicate = _duplicate(target.pid, descriptor, device, inode) + held.callback(os.close, duplicate) + credentials = _local_socket_peer(duplicate) + if credentials is not None and credentials["pid"] == issuer_pid: + if credentials["uid"] != issuer_uid: + raise AssertionError("the actual queued-grant sender has the wrong Unix owner") + matches.append((descriptor, device, inode, duplicate, credentials)) + if len(matches) != 1: + raise AssertionError("the stopped receiver does not retain one exact authenticated issuer socket") + return matches[0] + + def _expected_queued_grant(context: NativeNegativeContext, issuer_pid: int, issuer_start_token: str, target_pid: int, target_start_token: str) -> dict[str, object]: """Bind the expected packet to fixed consumer inputs and original durable custody.""" configuration = _consumer_configuration(context) @@ -557,24 +601,9 @@ def check_lifetimes() -> None: check_lifetimes() before = _descriptor_snapshot(target.pid) _stopped_receiver(target.pid, target.start_token, target.pidfd, before) - if len(before) > 4096: - raise AssertionError("the actual receiver descriptor inventory exceeds the fixture bound") expected = _expected_queued_grant(context, issuer_pid, issuer_start_token, target.pid, target.start_token) configuration = _consumer_configuration(context) - matches = [] - for descriptor, device, inode, _ in before: - if not stat.S_ISSOCK(Path(f"/proc/{target.pid}/fd/{descriptor}").stat().st_mode): - continue - duplicate = _duplicate(target.pid, descriptor, device, inode) - held.callback(os.close, duplicate) - credentials = _local_socket_peer(duplicate) - if credentials is not None and credentials["pid"] == issuer_pid: - if credentials["uid"] != configuration.issuer_uid: - raise AssertionError("the actual queued-grant sender has the wrong Unix owner") - matches.append((descriptor, device, inode, duplicate, credentials)) - if len(matches) != 1: - raise AssertionError("the stopped receiver does not retain one exact authenticated issuer socket") - descriptor, device, inode, duplicate, credentials = matches[0] + descriptor, device, inode, duplicate, credentials = _issuer_socket(target, issuer_pid, configuration.issuer_uid, held) encoded, flags_before, flags_after = _peek_queued_packet(duplicate, canonical(expected) + b"\n") check_lifetimes() _stopped_receiver(target.pid, target.start_token, target.pidfd, before) @@ -608,6 +637,83 @@ def check_lifetimes() -> None: } +@contextmanager +def observe_post_receipt_dispatch(context: NativeNegativeContext, issuer: subprocess.Popen, release: Callable[[], None]) -> Iterator[dict[str, object]]: + """Measure refusal or a real queued grant while the same receiver is stopped.""" + _require_publication_prefix(context, issuer.pid, "receipt") + with ExitStack() as held: + target = DockerTraceProcess(context) + held.callback(target.close) + issuer_start = _token(issuer.pid) + issuer_pidfd = os.pidfd_open(issuer.pid, 0) + held.callback(os.close, issuer_pidfd) + target.pause() + before = _descriptor_snapshot(target.pid) + _stopped_receiver(target.pid, target.start_token, target.pidfd, before) + configuration = _consumer_configuration(context) + descriptor, device, inode, duplicate, credentials = _issuer_socket(target, issuer.pid, configuration.issuer_uid, held) + flags = fcntl.fcntl(duplicate, fcntl.F_GETFL) + if _peek_pending_frame(duplicate): + raise AssertionError("the receiver already had a grant before the post-receipt issuer resumed") + observation: dict[str, object] = { + "schema": "actual-recovery-post-receipt-dispatch-v1", + "issuer_pid": issuer.pid, + "issuer_start_token": issuer_start, + "target_pid": target.pid, + "target_start_token": target.start_token, + "socket_descriptor": descriptor, + "socket_device": device, + "socket_inode": inode, + "peer_credentials": credentials, + "receiver_stopped": True, + "queue_empty_before_resume": True, + "queue_bytes_observed": 0, + "queued_grant": None, + "issuer_returncode": None, + "issuer_death_before_receiver_resumed": False, + } + try: + release() + poll = select.poll() + poll.register(issuer_pidfd, select.POLLIN) + poll.register(duplicate, select.POLLIN | select.POLLHUP) + deadline = time.monotonic() + 1800 + while True: + if time.monotonic() >= deadline: + raise AssertionError("the original issuer neither refused nor queued a complete post-receipt grant") + poll.poll(1000) + target.check() + _stopped_receiver(target.pid, target.start_token, target.pidfd, before) + queued = _peek_pending_frame(duplicate) + returncode = issuer.poll() + if queued: + observation["queued_grant"] = observe_queued_grant(context, issuer.pid, target_pid=target.pid, target_start_token=target.start_token, issuer_start_token=issuer_start) + observation["queue_bytes_observed"] = len(queued) + break + if returncode is not None: + if _peek_pending_frame(duplicate): + raise AssertionError("the original issuer left a grant before its observed exit") + observation["issuer_returncode"] = returncode + break + if fcntl.fcntl(duplicate, fcntl.F_GETFL) != flags or _local_socket_peer(duplicate) != credentials: + raise AssertionError("the exact receiver socket changed during the post-receipt observation") + observation["socket_flags_unchanged"] = True + _assert_no_target_admission(context, running_pid=target.pid) + yield observation + finally: + # A detected mutant's live issuer is terminated only after the + # actual unsafe dispatch evidence has been captured. No retry or + # original epoch repair is performed. + if issuer.poll() is None: + signal.pidfd_send_signal(issuer_pidfd, signal.SIGKILL) + issuer.wait(timeout=30) + dead = select.poll() + dead.register(issuer_pidfd, select.POLLIN) + if not dead.poll(0): + raise AssertionError("the original issuer remains alive before receiver resumption") + observation["issuer_death_before_receiver_resumed"] = True + + @contextmanager def pause_target_before_grant(context: NativeNegativeContext, issuer_pid: int) -> Iterator[dict[str, object]]: """Hold the actual receiver while the original issuer sends, then dies.""" diff --git a/scripts/recovery_vm/README.md b/scripts/recovery_vm/README.md index e7b118640..a7b06b5cc 100644 --- a/scripts/recovery_vm/README.md +++ b/scripts/recovery_vm/README.md @@ -30,6 +30,12 @@ another image or weaken the production source checks. Docker documents the [fresh-install storage default](https://docs.docker.com/engine/storage/containerd/) and the [cross-store image-ID behavior](https://github.com/docker/cli/issues/6670). +The guest installs the exact `uv==0.13.6` release and validates its reported release +number. The complete observed `--version` banner, including its bounded build +descriptor, is retained in `uv-version.txt` and supplied to staging and recovery +authorization. The production installer continues to require exact banner +equality; the fixture does not strip build identity from that comparison. + ## Reproduce a reviewed scenario Use a clean, committed checkout with the locked test dependencies installed. The @@ -64,7 +70,10 @@ scenario oracle. CI derives its matrix directly from this list and requires every matrix job to pass. The two `kill-after-*` debug aliases select boundaries already present in the matrix and do not add independent cases. All before/after publication cases and all eleven mutations after the original intent head run on -independent fresh VMs. They +independent fresh VMs. A distinct `post-receipt-ready-mutation` scenario changes +the exact READY receipt after consumption and receipt-head publication, then +resumes that same issuer and independently proves refusal with no queued grant. +They use real ptrace syscall stops around the actual exclusive record/head creation and successful file/directory fsync, with a pinned SIGKILL and no syscall-result replacement. The native process oracles also inspect real peer custody, original @@ -84,6 +93,29 @@ issuer's pidfd is dead before resuming the receiver; consumption must remain and the receiver must refuse current custody. The companion helper tests validate the fixture's native message parsing and guards; they are not VM acceptance. +## Verifier-omission control + +A separate CI job proves that the post-receipt scenario detects omission of only +the second final verification. It uses another fresh VM, copies the reviewed +project into a private control directory, and removes exactly that one call with +structural and byte-level checks. Its original producers, source identities, +derived images and issuer all use the same copied package before any history is +created. The canonical checkout and installed package remain unchanged. + +The control must reach the real receipt-head barrier and fail the unchanged +native oracle because an exact canonical grant is actually queued on the stopped +receiver's socket. An unrelated error, timeout, process exit or log message cannot +pass this control. The result has a separate `mutation-detected` status, binds the +exact original/mutant source evidence and preserves consumed state, the deliberate +READY mutation and unrelated workers. It supplies test-sensitivity evidence and +does not count as a successful production acceptance scenario. + +To prepare this explicitly selected control, use the reproduction command above +with `post-receipt-ready-mutation omit-post-consumption-verifier` as the final two +arguments to `prepare.sh`. Every ordinary preparation records `mutation_control` +as `none`. Both the complete unmodified production matrix and this separate +control must succeed for the aggregate CI gate. + The guest read credential is deleted on exit. The outer input credential must also be deleted after the run; the CI workflow does this even on failure. Artifacts contain the result, bounded diagnostics and guest/serial logs, not the diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh index e07351b20..33d275403 100644 --- a/scripts/recovery_vm/guest.sh +++ b/scripts/recovery_vm/guest.sh @@ -55,6 +55,7 @@ mkdir -m 700 /opt/recovery-fixture /opt/reviewed-source tar --extract --gzip --file "$input/source.tar.gz" --directory /opt/recovery-fixture cd /opt/recovery-fixture /opt/fixture-tools/bin/uv sync --locked --no-editable --no-dev --extra test --link-mode copy +verify_installed_watcher() { .venv/bin/python - <<'PY' from pathlib import Path import auto_coder.management.proxy_watch_cli as watcher @@ -65,6 +66,8 @@ identity = source.stat() assert identity.st_uid == 0 and identity.st_nlink == 1 print("Installed watcher source is a private, independently copied file.") PY +} +verify_installed_watcher docker compose version install -m 600 "$input/source-token" /run/recovery-fixture-source-token mapfile -t selected < <(python3 - "$input" <<'PY' @@ -73,10 +76,42 @@ import pathlib import sys manifest = json.loads((pathlib.Path(sys.argv[1]) / "manifest.json").read_text()) -for key in ("authorization_id", "manager_image", "proxy_image", "ready_source_sha", "scenario"): +for key in ("authorization_id", "manager_image", "proxy_image", "ready_source_sha", "scenario", "mutation_control"): print(manifest[key]) PY ) +driver_module=scripts.test_management_recovery_epoch_host +driver_prefix=() +case "${selected[5]}" in + none) ;; + omit-post-consumption-verifier) + test "${selected[4]}" = post-receipt-ready-mutation + .venv/bin/python -m scripts.recovery_epoch_mutation_control prepare \ + --source /opt/recovery-fixture \ + --destination /opt/recovery-mutation-control \ + --evidence "$output/mutation-source.json" + cd /opt/recovery-mutation-control + /opt/fixture-tools/bin/uv sync --locked --no-editable --no-dev --extra test --link-mode copy + verify_installed_watcher + driver_module=scripts.recovery_epoch_mutation_control + driver_prefix=(run --evidence "$output/mutation-source.json" --) + ;; + *) echo 'Unsupported recovery mutation control.' >&2; exit 2 ;; +esac +fixture_uv_version=$(.venv/bin/python - <<'PY' +import subprocess +from auto_coder.management.recovery_epoch_authorization import validate_uv_version + +banner = subprocess.check_output(["/opt/fixture-tools/bin/uv", "--version"], text=True).strip() +if not banner.startswith("uv "): + raise ValueError("The selected fixture uv did not return its version banner") +version = validate_uv_version(banner.removeprefix("uv ")) +if version.split(" ", 1)[0] != "0.13.6": + raise ValueError("The installed fixture uv does not match the reviewed release pin") +print(version) +PY +) +printf 'uv %s\n' "$fixture_uv_version" | tee "$output/uv-version.txt" git -C /opt/reviewed-source init git -C /opt/reviewed-source remote add origin https://github.com/kitamura-tetsuo/auto-coder.git git -C /opt/reviewed-source fetch --no-tags origin "${selected[3]}" @@ -111,11 +146,11 @@ PY export AUTOCODER_FRESH_TEST_BASE_IMAGE="${selected[1]}" export AUTOCODER_RECOVERY_CADDY_IMAGE="${selected[2]}" export PATH="/opt/fixture-tools/bin:$PATH" -.venv/bin/python -m scripts.test_management_recovery_epoch_host \ +.venv/bin/python -m "$driver_module" "${driver_prefix[@]}" \ --authorization-id "${selected[0]}" \ --source-repository /opt/reviewed-source \ --source-token-file /run/recovery-fixture-source-token \ --source-sha "${selected[3]}" \ --scenario "${selected[4]}" \ - --uv /opt/fixture-tools/bin/uv --uv-version 0.13.6 \ + --uv /opt/fixture-tools/bin/uv --uv-version "$fixture_uv_version" \ --output "$output/recovery-epoch-result.json" diff --git a/scripts/recovery_vm/prepare.sh b/scripts/recovery_vm/prepare.sh index 58613295d..04199ba8b 100644 --- a/scripts/recovery_vm/prepare.sh +++ b/scripts/recovery_vm/prepare.sh @@ -3,8 +3,8 @@ set -Eeuo pipefail umask 077 # Export exact reviewed source/images; preparation does not launch a guest. -if [[ "$#" != 6 || ! "$1" =~ ^/var/tmp/ac-recovery-vm-[a-zA-Z0-9_-]+$ || ! "$2" =~ ^[a-zA-Z0-9_.:-]{1,128}$ || ! "$3" =~ ^sha256:[0-9a-f]{64}$ || ! "$4" =~ ^sha256:[0-9a-f]{64}$ || ! "$6" =~ ^[a-z][a-z-]{1,63}$ ]]; then - echo 'Usage: prepare.sh /var/tmp/ac-recovery-vm-UNIQUE AUTHORIZATION_ID MANAGER_IMAGE_ID CADDY_IMAGE_ID READ_TOKEN_FILE SCENARIO' >&2 +if [[ ( "$#" != 6 && "$#" != 7 ) || ! "$1" =~ ^/var/tmp/ac-recovery-vm-[a-zA-Z0-9_-]+$ || ! "$2" =~ ^[a-zA-Z0-9_.:-]{1,128}$ || ! "$3" =~ ^sha256:[0-9a-f]{64}$ || ! "$4" =~ ^sha256:[0-9a-f]{64}$ || ! "$6" =~ ^[a-z][a-z-]{1,63}$ ]]; then + echo 'Usage: prepare.sh /var/tmp/ac-recovery-vm-UNIQUE AUTHORIZATION_ID MANAGER_IMAGE_ID CADDY_IMAGE_ID READ_TOKEN_FILE SCENARIO [omit-post-consumption-verifier]' >&2 exit 2 fi fixture_root=$1 @@ -13,6 +13,11 @@ manager_image=$3 proxy_image=$4 source_token_file=$5 fixture_scenario=$6 +fixture_mutation_control=${7:-none} +if [[ "$fixture_mutation_control" != none && ( "$fixture_mutation_control" != omit-post-consumption-verifier || "$fixture_scenario" != post-receipt-ready-mutation ) ]]; then + echo 'Only the explicit post-receipt verifier-omission control is supported.' >&2 + exit 2 +fi tooling_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) repo_root=$(git -C "$tooling_root" rev-parse --show-toplevel) cd -- "$repo_root" @@ -69,7 +74,7 @@ docker image save "$manager_image" "$proxy_image" | tee "$fixture_root/input/ima cp "$tooling_root/guest.sh" "$fixture_root/input/guest.sh" install -m 600 "$source_token_file" "$fixture_root/input/source-token" fixture_commit=$(git -C "$repo_root" rev-parse HEAD) -python3 - "$fixture_root" "$fixture_authorization" "$fixture_commit" "$manager_image" "$proxy_image" "$fixture_scenario" "$fixture_accelerator" "$fixture_image_store" <<'PY' +python3 - "$fixture_root" "$fixture_authorization" "$fixture_commit" "$manager_image" "$proxy_image" "$fixture_scenario" "$fixture_accelerator" "$fixture_image_store" "$fixture_mutation_control" <<'PY' import hashlib import json import pathlib @@ -77,7 +82,7 @@ import sys root = pathlib.Path(sys.argv[1]) inputs = root / "input" -manifest = dict(zip(("authorization_id", "commit", "manager_image", "proxy_image", "scenario", "accelerator", "image_store"), sys.argv[2:])) +manifest = dict(zip(("authorization_id", "commit", "manager_image", "proxy_image", "scenario", "accelerator", "image_store", "mutation_control"), sys.argv[2:])) manifest["schema"] = "auto-coder-recovery-vm-input-v1" manifest["ready_source_sha"] = "e742338b3f311e6d30a22a8f5341b8072513d813" manifest["inputs_sha256"] = {} diff --git a/scripts/recovery_vm/runner.sh b/scripts/recovery_vm/runner.sh index c05521543..ee2fe852c 100644 --- a/scripts/recovery_vm/runner.sh +++ b/scripts/recovery_vm/runner.sh @@ -42,11 +42,34 @@ if [[ "$result" != 0 ]]; then exit 1 fi python3 - <<'PY' +import hashlib import json from pathlib import Path result = json.loads(Path("/output/recovery-epoch-result.json").read_text()) manifest = json.loads(Path("/input/manifest.json").read_text()) -if result.get("status") != "passed" or result.get("scenario") != manifest["scenario"]: - raise SystemExit("The native recovery result does not record complete acceptance") +if result.get("scenario") != manifest["scenario"]: + raise SystemExit("The recovery result does not match the prepared scenario") +control = manifest["mutation_control"] +if control == "none": + if result.get("schema") != "auto-coder-native-recovery-epoch-result-v1" or result.get("status") != "passed": + raise SystemExit("The native recovery result does not record complete acceptance") +elif control == "omit-post-consumption-verifier": + evidence = Path("/output/mutation-source.json").read_bytes() + if result.get("schema") != "auto-coder-native-recovery-mutation-control-result-v1" or result.get("status") != "mutation-detected" or result.get("control") != control or result.get("source_evidence_sha256") != hashlib.sha256(evidence).hexdigest(): + raise SystemExit("The separate mutation-control result lacks its exact source evidence") + oracle = result.get("oracle") + if not isinstance(oracle, dict) or oracle.get("name") != "post-receipt-grant-forbidden" or oracle.get("detected") is not True: + raise SystemExit("The mutation control did not detect the specified native regression") + queued = oracle.get("queued_grant") + if not isinstance(queued, dict) or queued.get("schema") != "actual-recovery-queued-grant-v1" or not isinstance(queued.get("grant"), dict): + raise SystemExit("The mutation control did not independently observe a real queued grant") + frame = json.dumps(queued["grant"], sort_keys=True, separators=(",", ":")).encode() + b"\n" + if queued.get("message_sha256") != hashlib.sha256(frame).hexdigest() or queued.get("bytes_observed") != len(frame) or queued.get("socket_flags_before") != queued.get("socket_flags_after"): + raise SystemExit("The mutation-control queued grant is incomplete or changed the socket") + for fact in ("peek_only", "socket_flags_unchanged", "bytes_still_queued", "receiver_stopped", "issuer_alive"): + if queued.get(fact) is not True: + raise SystemExit("The mutation-control observation omitted " + fact) +else: + raise SystemExit("The prepared recovery control is unsupported") PY diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index b41adebbf..43fabdf73 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -30,8 +30,9 @@ from scripts.recovery_epoch_containment_fixture import CapturedGeneration, capture_generation, enroll_recovery_scope, selected_file, start_watcher, target_from_container from scripts.recovery_epoch_fault_tracer import BOUNDARY_SCENARIOS, PUBLICATIONS, KernelPublicationBarrier, PublicationObservation, parse_boundary +from scripts.recovery_epoch_mutation_control import CONTROL, CONTROL_SCHEMA, ORACLE, PostReceiptGrantDispatched, assert_post_receipt_refusal, validate_control_result, verify_source_evidence from scripts.recovery_epoch_native_negatives import REQUIRED_NEGATIVE_CASES, REQUIRED_REPORT_CASES, NativeNegativeContext, _replacement, assert_unreserved_epoch, run_native_negatives, verify_native_containment_reports -from scripts.recovery_epoch_native_peer import LATE_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, mutate_after_intent, observe_queued_grant, pause_target_before_grant, run_native_peer_cases +from scripts.recovery_epoch_native_peer import LATE_MUTATION_SCENARIOS, POST_RECEIPT_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, mutate_after_publication, observe_post_receipt_dispatch, observe_queued_grant, pause_target_before_grant, run_native_peer_cases from scripts.test_management_containment_closed_host import InhibitedStockRoutes, WatcherFixture, require_disposable_host from scripts.test_management_containment_enrollment_host import DisposableSystemdRoutes, _private_directory, _write_private @@ -44,7 +45,7 @@ from auto_coder.management.original_attempt_evidence import derive_lineage, observe, verify_lineage from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, source_identity from auto_coder.management.recovery_containment_model import RecoveryWatcher -from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, AUTHORIZATION_ROOT, AUTHORIZATION_SCHEMA, PEER_ROOT, PEER_SOCKET, EpochAuthorization +from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, AUTHORIZATION_ROOT, AUTHORIZATION_SCHEMA, PEER_ROOT, PEER_SOCKET, EpochAuthorization, validate_uv_version from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA, ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource from auto_coder.management.recovery_epoch_state import EPOCH_ROOT, EpochInventory from auto_coder.management.self_update_handoff import runtime_for_receipt @@ -59,7 +60,7 @@ LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused", "cross-deployment-original-lineage-refused"}) FACTORY_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES if case.startswith("factory-")) TASK_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES | REQUIRED_REPORT_CASES if "nonleader" in case) -ACCEPTANCE_SCENARIOS = ("grant", "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS, *LATE_MUTATION_SCENARIOS) +ACCEPTANCE_SCENARIOS = ("grant", "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS, *LATE_MUTATION_SCENARIOS, *POST_RECEIPT_MUTATION_SCENARIOS) SCENARIOS = (*ACCEPTANCE_SCENARIOS, "kill-after-intent", "kill-after-consume") REQUIRED_CASES = frozenset( { @@ -582,12 +583,24 @@ def validate_result(value: dict[str, object]) -> None: elif scenario == "retarget-after-reservation": assert value["durable"]["phase"] == "reserved" and value["durable"]["consumer_grant_acknowledged"] is False assert value["durable"]["inventory_sha256"] != value["durable"]["retargeted_inventory_sha256"] - elif scenario in LATE_MUTATION_SCENARIOS: - assert value["durable"]["phase"] == "intent" and value["durable"]["consumer_grant_acknowledged"] is False and value["durable"]["grant_send_completed"] is False - assert value["durable"]["mutation"]["scenario"] == scenario and value["durable"]["mutation"]["boundary"] == "after-intent-head" + elif scenario in (*LATE_MUTATION_SCENARIOS, *POST_RECEIPT_MUTATION_SCENARIOS): + phase = "receipt" if scenario in POST_RECEIPT_MUTATION_SCENARIOS else "intent" + assert value["durable"]["phase"] == phase and value["durable"]["consumer_grant_acknowledged"] is False and value["durable"]["grant_send_completed"] is False + assert value["durable"]["mutation"]["scenario"] == scenario and value["durable"]["mutation"]["boundary"] == "after-" + phase + "-head" assert value.get("preservation_baseline") == "after-deliberate-fixture-mutation" assert value["durable"]["mutation"]["preservation_baseline"] == value["preservation_baseline"] assert value["preserved_originals_sha256"] == value["durable"]["originals_after_injection_sha256"] + if scenario in POST_RECEIPT_MUTATION_SCENARIOS: + durable = value["durable"] + assert_post_receipt_refusal(durable["dispatch"]) + assert durable["dispatch"]["issuer_death_before_receiver_resumed"] is True and durable["replay_returncode"] == 2 + assert durable["retained_publications"] == list(PUBLICATIONS) + assert durable["epoch_sha256_before_resume"] == durable["epoch_sha256_after_refusal"] == durable["epoch_sha256_after_replay"] + boundary = durable["publication_boundary"] + assert (boundary["name"], boundary["when"], boundary["file_fsync_observed"], boundary["directory_fsync_observed"]) == ("receipt-head.json", "after", True, True) + assert all(durable["mutation"][name] == durable["dispatch"][name] for name in ("issuer_pid", "issuer_start_token")) + changed = durable["mutation"]["file_fault"] + assert tuple(changed["before"][:2]) == tuple(changed["after"][:2]) and changed["before"][-1] != changed["after"][-1] if scenario == "late-target-loss": target_fault = value["durable"]["mutation"]["target_fault"] assert target_fault["pidfd_death_observed"] is True and target_fault["exact_docker_id_absent"] is True and target_fault["docker_rm_returncode"] == 0 @@ -625,6 +638,8 @@ def _selected_boundary(scenario: str) -> tuple[str, str]: return parse_boundary(scenario) if scenario in LATE_MUTATION_SCENARIOS or scenario == "kill-after-intent": return "intent-head.json", "after" + if scenario in POST_RECEIPT_MUTATION_SCENARIOS: + return "receipt-head.json", "after" if scenario == "kill-after-consume": return "consumed-head.json", "after" if scenario == "lost-reply": @@ -804,7 +819,7 @@ def _late_mutation( ) -> tuple[dict[str, object], str]: """Require an actual late refusal while retaining the injected uncertainty.""" original = _snapshot(retained_roots) - with mutate_after_intent(context, scenario, issuer.pid) as observation: + with mutate_after_publication(context, scenario, issuer.pid) as observation: injected = _snapshot(retained_roots) epoch = _snapshot((EPOCH_ROOT,)) check_workers() @@ -838,10 +853,79 @@ def _late_mutation( return result, injected -def run_fixture(*, authorization_id: str, source_repository: Path, source_token_file: Path, source_sha: str, uv: Path, uv_version: str, output: Path, scenario: str = "grant") -> dict[str, object]: +def _post_receipt_mutation( + context: NativeNegativeContext, + scenario: str, + barrier: KernelPublicationBarrier, + observed: PublicationObservation, + issuer: subprocess.Popen, + issuer_log: Path, + consumer_log: Path, + target: PreparedTarget, + retained_roots: tuple[Path, ...], + check_workers: Callable[[], None], + *, + mutation_control: bool, +) -> tuple[dict[str, object], str]: + """Require the same issuer to reject stale READY after its durable receipt.""" + assert (observed.name, observed.when, observed.file_fsync_observed, observed.directory_fsync_observed) == ("receipt-head.json", "after", True, True) + original = _snapshot(retained_roots) + detected = False + with mutate_after_publication(context, scenario, issuer.pid) as mutation: + injected = _snapshot(retained_roots) + epoch = _snapshot((EPOCH_ROOT,)) + assert set(os.listdir(EPOCH_ROOT)) == {"inventory.json", "epoch.lock", *PUBLICATIONS} + check_workers() + with observe_post_receipt_dispatch(context, issuer, barrier.release) as dispatch: + try: + assert_post_receipt_refusal(dispatch) + except PostReceiptGrantDispatched: + if not mutation_control: + raise + detected = True + if mutation_control and not detected: + raise AssertionError("the focused second-verifier mutation did not fail the actual no-grant oracle") + _wait(lambda: Chain.docker("inspect", "--format", "{{.State.Running}}", target.generation.container) == "false", "post-receipt refusal ends the original stopped receiver", timeout=120) + assert Chain.docker("inspect", "--format", "{{.State.ExitCode}}", target.generation.container) == "1" + _no_target_child(target.generation, running=False) + assert "Recovery custody acknowledged" not in consumer_log.read_text() + if not detected: + assert "One consumed recovery epoch grant send completed" not in issuer_log.read_text() + after_refusal = _snapshot((EPOCH_ROOT,)) + assert after_refusal == epoch and _snapshot(retained_roots) == injected + check_workers() + replay = subprocess.run([sys.executable, "-B", "-m", "auto_coder.management.recovery_epoch"], capture_output=True, text=True, timeout=30) + after_replay = _snapshot((EPOCH_ROOT,)) + assert replay.returncode == 2 and after_replay == epoch and _snapshot(retained_roots) == injected + check_workers() + result = { + "phase": "receipt", + "consumer_grant_acknowledged": False, + "grant_send_completed": detected, + "mutation": mutation, + "dispatch": dispatch, + "publication_boundary": asdict(observed), + "retained_publications": list(PUBLICATIONS), + "replay_returncode": replay.returncode, + "originals_before_injection_sha256": original, + "originals_after_injection_sha256": injected, + "epoch_sha256_before_resume": epoch, + "epoch_sha256_after_refusal": after_refusal, + "epoch_sha256_after_replay": after_replay, + } + return result, injected + + +def run_fixture(*, authorization_id: str, source_repository: Path, source_token_file: Path, source_sha: str, uv: Path, uv_version: str, output: Path, scenario: str = "grant", mutation_control_manifest: Path | None = None) -> dict[str, object]: required_cases(scenario) - if not re.fullmatch(r"[0-9a-f]{40}", source_sha) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", uv_version): - raise ValueError("an exact qualified source and installer version are required") + source_mutation = None + if mutation_control_manifest is not None: + if scenario not in POST_RECEIPT_MUTATION_SCENARIOS: + raise ValueError("the focused mutation control requires its exact post-receipt scenario") + source_mutation = verify_source_evidence(mutation_control_manifest, executing=True) + if not re.fullmatch(r"[0-9a-f]{40}", source_sha): + raise ValueError("an exact qualified source is required") + validate_uv_version(uv_version) if any(not path.is_absolute() or path != path.resolve(strict=True) for path in (source_repository, source_token_file, uv)) or not output.is_absolute(): raise ValueError("canonical absolute native fixture inputs are required") for name in ("AUTOCODER_FRESH_TEST_BASE_IMAGE", "AUTOCODER_RECOVERY_CADDY_IMAGE"): @@ -864,6 +948,8 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ setup_logger(stream=sys.stdout, log_file=str(root / "fixture.log")) cases: list[str] = [] result: dict[str, object] = {"schema": RESULT_SCHEMA, "status": "failed", "scenario": scenario, "cases": cases, "authorization_id": authorization_id, "source_identity": source_identity(), "no_child_started": False, "preserved_originals": False, "preserved_workers": False} + if source_mutation is not None: + result.update(schema=CONTROL_SCHEMA, control=CONTROL, source_mutation=source_mutation[0], source_evidence_sha256=source_mutation[1]) chain = None workers = None stock = None @@ -989,6 +1075,9 @@ def preservation() -> object: elif scenario in LATE_MUTATION_SCENARIOS: result["durable"], preserved = _late_mutation(native_context, scenario, barrier, issuer, issuer_log, consumer_log, target, retained_roots, lambda: workers.preserved(*worker_before)) result["preservation_baseline"] = "after-deliberate-fixture-mutation" + elif scenario in POST_RECEIPT_MUTATION_SCENARIOS: + result["durable"], preserved = _post_receipt_mutation(native_context, scenario, barrier, observed_boundary, issuer, issuer_log, consumer_log, target, retained_roots, lambda: workers.preserved(*worker_before), mutation_control=source_mutation is not None) + result["preservation_baseline"] = "after-deliberate-fixture-mutation" else: barrier.kill() actual = scenario if scenario in BOUNDARY_SCENARIOS else "after-" + publication.removesuffix(".json") @@ -998,6 +1087,23 @@ def preservation() -> object: cases.extend(("native-" + scenario, "interrupted-epoch-replay-refused")) assert _snapshot(retained_roots) == preserved workers.preserved(*worker_before) + if mutation_control_manifest is not None: + cases.extend(("originals-oauth-and-workers-preserved", "no-consumer-child-or-admission")) + result.update( + status="mutation-detected", + no_child_started=True, + preserved_originals=True, + preserved_workers=True, + preserved_originals_sha256=preserved, + preserved_worker_claims_sha256=worker_claims, + worker_lifetimes=worker_baseline, + worker_effect_posts=worker_posts, + oracle={"name": ORACLE, "detected": True, "queued_grant": result["durable"]["dispatch"]["queued_grant"]}, + ) + assert set(cases) == required_cases(scenario) + validate_control_result(result, mutation_control_manifest) + logger.info("The focused copied-source mutant failed the actual {} oracle; production acceptance is separate", ORACLE) + return result _passed(result, cases, preserved, worker_claims, worker_baseline, worker_posts) return result _wait(lambda: (EPOCH_ROOT / "receipt-head.json").is_file() and consumer_log.is_file() and "Recovery custody acknowledged" in consumer_log.read_text(), "native grant and consumer custody", timeout=1800, process=issuer) @@ -1096,6 +1202,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--uv-version", required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--scenario", choices=SCENARIOS, default="grant") + parser.add_argument("--mutation-control-manifest", type=Path) args = parser.parse_args(argv) run_fixture(**vars(args)) return 0 diff --git a/src/auto_coder/management/recovery_epoch_authorization.py b/src/auto_coder/management/recovery_epoch_authorization.py index 75e796a6b..09c37a65e 100644 --- a/src/auto_coder/management/recovery_epoch_authorization.py +++ b/src/auto_coder/management/recovery_epoch_authorization.py @@ -32,6 +32,14 @@ AUTHORIZATION_SCHEMA = "management-recovery-authorization-v1" _SHA = re.compile(r"[0-9a-f]{40}\Z", re.ASCII) _TOKEN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z", re.ASCII) +_UV_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?: \([A-Za-z0-9_.-](?:[A-Za-z0-9_. -]{0,126}[A-Za-z0-9_.-])?\))?\Z", re.ASCII) + + +def validate_uv_version(value: object) -> str: + """Retain the exact installer pin, including its bounded build descriptor.""" + if not isinstance(value, str) or len(value) > 128 or not _UV_VERSION.fullmatch(value): + raise EpochUnavailable("exact configured installer version required") + return value @dataclass(frozen=True) @@ -79,8 +87,7 @@ def parse(cls, value: object) -> EpochAuthorization: raise EpochUnavailable("history names another recovery deployment") for path in (result.history.root, result.source_repository, result.source_token_file, result.python_executable, result.uv_executable): _path(path) - if not isinstance(result.uv_version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", result.uv_version): - raise EpochUnavailable("exact configured installer version required") + validate_uv_version(result.uv_version) for field in ("socket_directory_device", "socket_directory_inode"): if type(getattr(result, field)) is not int or getattr(result, field) <= 0: raise EpochUnavailable("fixed recovery peer directory identity required") diff --git a/tests/test_management_recovery_epoch_authorization.py b/tests/test_management_recovery_epoch_authorization.py index d0cb2f696..50f2fd7b9 100644 --- a/tests/test_management_recovery_epoch_authorization.py +++ b/tests/test_management_recovery_epoch_authorization.py @@ -13,7 +13,7 @@ from auto_coder.management.original_attempts import HistoryBinding, canonical from auto_coder.management.recovery_containment_model import RecoveryFile -from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_SCHEMA, EpochAuthorization, _load_authorization, load_authorization, read_bound_file +from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_SCHEMA, EpochAuthorization, _load_authorization, load_authorization, read_bound_file, validate_uv_version from auto_coder.management.recovery_epoch_state import EpochBinding, EpochInventory, EpochUnavailable, _hold_recovery_epoch, issuer_lifetime pytestmark = pytest.mark.usefixtures("_use_real_commands", "_use_real_sleep") @@ -122,6 +122,59 @@ def test_complete_authorization_is_bound_to_fixed_inventory(tmp_path): assert tuple(inspect.signature(load_authorization).parameters) == ("inventory",) +@pytest.mark.parametrize("version", ["0.7.22", "0.13.6 (329bf94 2026-10-26)", "0.13.6 (x86_64-unknown-linux-gnu)", "0.13.6 (x)", "0.13.6 (" + "x" * 119 + ")"]) +def test_authorization_preserves_the_complete_installer_pin(tmp_path, version): + _, path, authorization, inventory = _fixture(tmp_path) + selected = replace(authorization, uv_version=version) + encoded = canonical(asdict(selected)) + path.write_bytes(encoded) + observed = _load_authorization(path, replace(inventory, authorization_sha256=hashlib.sha256(encoded).hexdigest())) + assert validate_uv_version(version) == version + assert observed.authorization == selected + assert observed.authorization.uv_version == version + assert observed.original.sha256 != inventory.authorization_sha256 or version == authorization.uv_version + + +@pytest.mark.parametrize( + "version", + [ + None, + True, + 13, + "", + "0.13", + "uv 0.13.6", + "0.13.6\n", + "0.13.6\r", + "0.13.6 ", + "0.13.6 (x) extra", + "0.13.6 (x) (y)", + "0.13.6 ()", + "0.13.6 ( )", + "0.13.6 ( x)", + "0.13.6 (x )", + "0.13.6 (x\ny)", + "0.13.6 (x\ty)", + "0.13.6 (nested(x))", + "0.13.6 ($HOME)", + "0.13.6 (`id`)", + "0.13.6 (x;y)", + "0.13.6 (x/y)", + "0.13.6 (é)", + "０.１３.６", + "0.13.6 (" + "x" * 120 + ")", + ], +) +def test_authorization_rejects_ambiguous_or_unbounded_installer_pin(tmp_path, version): + _, _, authorization, _ = _fixture(tmp_path) + raw = asdict(authorization) + raw["uv_version"] = version + with pytest.raises(EpochUnavailable, match="exact configured installer version"): + EpochAuthorization.parse(raw) + with pytest.raises(EpochUnavailable, match="exact configured installer version"): + validate_uv_version(version) + + @pytest.mark.parametrize("field,value", [("authorization_id", "other-review"), ("request_id", "other-request"), ("preparation_id", "other-preparation"), ("target_id", "f" * 64), ("deployment_id", "other-deployment"), ("enrollment_ids", ("another-scope",)), ("authorization_sha256", "f" * 64)]) def test_authorization_cannot_retarget_an_inventory(tmp_path, field, value): root, path, _, inventory = _fixture(tmp_path) diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index 4ae2a3a71..a55272249 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -348,8 +348,10 @@ def test_unknown_native_scenario_is_not_a_success_alias() -> None: def test_canonical_acceptance_contains_all_distinct_publication_and_late_faults() -> None: assert len(set(fixture.ACCEPTANCE_SCENARIOS)) == len(fixture.ACCEPTANCE_SCENARIOS) - assert set(fixture.ACCEPTANCE_SCENARIOS) == {"grant", "lost-reply", "retarget-after-reservation", *fixture.BOUNDARY_SCENARIOS, *fixture.LATE_MUTATION_SCENARIOS} + assert set(fixture.ACCEPTANCE_SCENARIOS) == {"grant", "lost-reply", "retarget-after-reservation", *fixture.BOUNDARY_SCENARIOS, *fixture.LATE_MUTATION_SCENARIOS, *fixture.POST_RECEIPT_MUTATION_SCENARIOS} assert set(fixture.SCENARIOS) - set(fixture.ACCEPTANCE_SCENARIOS) == {"kill-after-intent", "kill-after-consume"} + assert fixture._selected_boundary("post-receipt-ready-mutation") == ("receipt-head.json", "after") + assert all(fixture._selected_boundary(name) == ("intent-head.json", "after") for name in fixture.LATE_MUTATION_SCENARIOS) assert fixture._selected_boundary("kill-after-intent") == fixture._selected_boundary("after-intent-head") == ("intent-head.json", "after") assert fixture._selected_boundary("kill-after-consume") == fixture._selected_boundary("after-consumed-head") == ("consumed-head.json", "after") assert fixture._selected_boundary("lost-reply") == ("receipt-head.json", "after") @@ -466,6 +468,77 @@ def test_late_mutation_result_names_its_deliberate_preservation_boundary(scenari fixture.validate_result(result) +def _post_receipt_result() -> dict[str, object]: + result = _result() + scenario = "post-receipt-ready-mutation" + result.update( + scenario=scenario, + cases=sorted(fixture.required_cases(scenario)), + preservation_baseline="after-deliberate-fixture-mutation", + preserved_originals_sha256="c" * 64, + durable={ + "phase": "receipt", + "consumer_grant_acknowledged": False, + "grant_send_completed": False, + "mutation": {"scenario": scenario, "boundary": "after-receipt-head", "preservation_baseline": "after-deliberate-fixture-mutation", "issuer_pid": 123, "issuer_start_token": "456", "file_fault": {"before": [1, 2, "a" * 64], "after": [1, 2, "b" * 64]}}, + "originals_after_injection_sha256": "c" * 64, + "dispatch": { + "schema": "actual-recovery-post-receipt-dispatch-v1", + "issuer_pid": 123, + "issuer_start_token": "456", + "receiver_stopped": True, + "queue_empty_before_resume": True, + "socket_flags_unchanged": True, + "queue_bytes_observed": 0, + "queued_grant": None, + "issuer_returncode": 2, + "issuer_death_before_receiver_resumed": True, + }, + "replay_returncode": 2, + "retained_publications": list(fixture.PUBLICATIONS), + "epoch_sha256_before_resume": "d" * 64, + "epoch_sha256_after_refusal": "d" * 64, + "epoch_sha256_after_replay": "d" * 64, + "publication_boundary": {"name": "receipt-head.json", "when": "after", "file_fsync_observed": True, "directory_fsync_observed": True}, + }, + ) + return result + + +def test_post_receipt_mutation_result_requires_current_refusal_and_consumed_history() -> None: + fixture.validate_result(_post_receipt_result()) + + +@pytest.mark.parametrize("mutation", ["intent-boundary", "no-fsync", "lost-receipt", "repaired-epoch", "replayed-epoch", "replay-accepted", "issuer-killed", "queued-data", "receiver-running", "unchanged-ready", "foreign-issuer"]) +def test_post_receipt_result_cannot_substitute_kill_only_or_earlier_boundary(mutation: str) -> None: + result = _post_receipt_result() + durable = result["durable"] + if mutation == "intent-boundary": + durable["publication_boundary"]["name"] = "intent-head.json" + elif mutation == "no-fsync": + durable["publication_boundary"]["directory_fsync_observed"] = False + elif mutation == "lost-receipt": + durable["retained_publications"].pop() + elif mutation == "repaired-epoch": + durable["epoch_sha256_after_refusal"] = "e" * 64 + elif mutation == "replayed-epoch": + durable["epoch_sha256_after_replay"] = "e" * 64 + elif mutation == "replay-accepted": + durable["replay_returncode"] = 0 + elif mutation == "issuer-killed": + durable["dispatch"]["issuer_returncode"] = -9 + elif mutation == "queued-data": + durable["dispatch"]["queue_bytes_observed"] = 1 + elif mutation == "receiver-running": + durable["dispatch"]["receiver_stopped"] = False + elif mutation == "unchanged-ready": + durable["mutation"]["file_fault"]["after"] = durable["mutation"]["file_fault"]["before"] + else: + durable["dispatch"]["issuer_start_token"] = "999" + with pytest.raises(AssertionError): + fixture.validate_result(result) + + @pytest.mark.parametrize("field", ["pidfd_death_observed", "exact_docker_id_absent", "other_docker_ids_preserved"]) def test_late_target_loss_requires_death_absence_and_preserved_other_objects(field: str) -> None: scenario = "late-target-loss" diff --git a/tests/test_management_recovery_epoch_mutation_control.py b/tests/test_management_recovery_epoch_mutation_control.py new file mode 100644 index 000000000..635857229 --- /dev/null +++ b/tests/test_management_recovery_epoch_mutation_control.py @@ -0,0 +1,186 @@ +"""Bounded guards for the separate real-VM second-verifier mutation control. + +These checks validate transformation and result parsing, not native acceptance. +The dedicated disposable VM must execute the same no-grant oracle with genuine +sources, original records, Docker processes and actual queued receiver bytes. +""" + +from __future__ import annotations + +import ast +import hashlib +import json +from pathlib import Path + +import pytest +from scripts import recovery_epoch_mutation_control as control + +from auto_coder.management import recovery_epoch + + +def _original() -> bytes: + return Path(recovery_epoch.__file__).read_bytes() + + +def test_real_issuer_mutation_removes_only_the_second_post_consumption_verifier() -> None: + source = _original() + mutant, line = control.omit_post_consumption_verifier(source) + assert source.splitlines()[line - 1].strip() == b"final_verification()" + assert mutant == b"".join(source.splitlines(keepends=True)[: line - 1] + source.splitlines(keepends=True)[line:]) + function = next(node for node in ast.parse(mutant).body if isinstance(node, ast.FunctionDef) and node.name == "issue_epoch_once") + calls = [node for node in ast.walk(function) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "final_verification"] + assert len(calls) == 1 + consume = next(node for node in ast.walk(function) if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "consume") + assert calls[0].lineno < consume.lineno + assert b"session.check(custody)" in mutant and b"session.connection.sendall(packet)" in mutant + + +@pytest.mark.parametrize("change", ["already-removed", "extra-call", "wrong-consume", "argument", "wrong-function"]) +def test_transform_refuses_other_issuer_shapes(change: str) -> None: + source = _original() + if change == "already-removed": + source = control.omit_post_consumption_verifier(source)[0] + elif change == "extra-call": + source = source.replace(b" final_verification()", b" final_verification()\n final_verification()", 1) + elif change == "wrong-consume": + source = source.replace(b"receipt = state.consume(binding)", b"receipt = state.reserve(binding)") + elif change == "argument": + source = source.replace(b" final_verification()", b" final_verification(binding)", 1) + else: + source = source.replace(b"def issue_epoch_once(", b"def another_issuer(") + with pytest.raises(ValueError): + control.omit_post_consumption_verifier(source) + + +def _copy(tmp_path: Path) -> tuple[Path, Path, Path]: + source, destination, evidence = tmp_path / "original", tmp_path / "control", tmp_path / "mutation-source.json" + selected = source / control.ISSUER_SOURCE + selected.parent.mkdir(parents=True) + selected.write_bytes(_original()) + (source / "unchanged.txt").write_bytes(b"original project bytes") + (source / ".venv").mkdir() + (source / ".venv" / "not-copied").write_bytes(b"environment") + return source, destination, evidence + + +def test_private_copy_retains_original_and_all_other_project_files(tmp_path: Path) -> None: + source, destination, evidence = _copy(tmp_path) + original = (source / control.ISSUER_SOURCE).read_bytes() + value = control.prepare(source, destination, evidence) + assert (source / control.ISSUER_SOURCE).read_bytes() == original + assert (destination / "unchanged.txt").read_bytes() == b"original project bytes" + assert not (destination / ".venv").exists() + observed, digest = control.verify_source_evidence(evidence) + assert observed == value and digest == hashlib.sha256(evidence.read_bytes()).hexdigest() + assert value["original_source_sha256"] != value["mutant_source_sha256"] + assert value["original_project_sha256"] != value["mutant_project_sha256"] + assert (evidence.stat().st_mode & 0o777) == 0o600 + + +@pytest.mark.parametrize("target", ["source", "copy", "evidence"]) +def test_source_evidence_refuses_changed_original_mutant_or_claim(tmp_path: Path, target: str) -> None: + source, destination, evidence = _copy(tmp_path) + control.prepare(source, destination, evidence) + if target == "evidence": + value = json.loads(evidence.read_bytes()) + value["first_verifier_retained"] = False + evidence.write_bytes(control._canonical(value)) + else: + path = (source if target == "source" else destination) / control.ISSUER_SOURCE + path.write_bytes(path.read_bytes() + b"\n# changed original source\n") + with pytest.raises(AssertionError): + control.verify_source_evidence(evidence) + + +def test_copy_refuses_existing_destination_without_changing_it(tmp_path: Path) -> None: + source, destination, evidence = _copy(tmp_path) + destination.mkdir() + retained = destination / "retained" + retained.write_bytes(b"existing") + with pytest.raises(ValueError, match="new separate"): + control.prepare(source, destination, evidence) + assert retained.read_bytes() == b"existing" and not evidence.exists() + + +def _empty_dispatch() -> dict[str, object]: + return {"schema": "actual-recovery-post-receipt-dispatch-v1", "receiver_stopped": True, "queue_empty_before_resume": True, "socket_flags_unchanged": True, "queue_bytes_observed": 0, "queued_grant": None, "issuer_returncode": 2} + + +def test_correct_post_receipt_refusal_passes_the_shared_oracle() -> None: + control.assert_post_receipt_refusal(_empty_dispatch()) + + +@pytest.mark.parametrize("field,value", [("issuer_returncode", None), ("issuer_returncode", 0), ("issuer_returncode", -9), ("queue_bytes_observed", 1), ("receiver_stopped", False), ("queue_empty_before_resume", False), ("socket_flags_unchanged", False)]) +def test_generic_failure_kill_timeout_or_unsupported_queue_is_not_mutation_detection(field: str, value: object) -> None: + observation = _empty_dispatch() + observation[field] = value + with pytest.raises(AssertionError) as caught: + control.assert_post_receipt_refusal(observation) + assert type(caught.value) is not control.PostReceiptGrantDispatched + + +def _queued_dispatch() -> dict[str, object]: + issuer = {"issuer_pid": 123, "issuer_start_token": "345", "target_pid": 678, "target_start_token": "901", "peer_credentials": {"pid": 123, "uid": 0, "gid": 0}} + grant = { + "schema": "management-recovery-epoch-grant-v1", + "granted": "recovery-epoch-consumed", + "request_id": "fixture-request", + "authorization_id": "fixture-authorization", + "target_id": "a" * 64, + "challenge": "b" * 64, + "receipt_nonce": "c" * 64, + "inventory_sha256": "d" * 64, + "binding_sha256": "e" * 64, + "consumed_sha256": "f" * 64, + "receipt_sha256": "0" * 64, + "host_boot_id": "actual-component-boot", + "host_pid": issuer["issuer_pid"], + "host_start_token": issuer["issuer_start_token"], + } + encoded = control._canonical(grant) + b"\n" + queued = { + **issuer, + "schema": "actual-recovery-queued-grant-v1", + "grant_schema": grant["schema"], + "target_id": grant["target_id"], + "grant": grant, + "bytes_observed": len(encoded), + "message_sha256": hashlib.sha256(encoded).hexdigest(), + "socket_flags_before": 2, + "socket_flags_after": 2, + "socket_inode": 77, + "peek_only": True, + "socket_flags_unchanged": True, + "bytes_still_queued": True, + "receiver_stopped": True, + "issuer_alive": True, + } + return {**_empty_dispatch(), **issuer, "queued_grant": queued, "queue_bytes_observed": len(encoded), "issuer_returncode": None} + + +def test_only_exact_original_queued_grant_triggers_named_native_failure() -> None: + with pytest.raises(control.PostReceiptGrantDispatched, match=control.ORACLE): + control.assert_post_receipt_refusal(_queued_dispatch()) + + +@pytest.mark.parametrize("mutation", ["digest", "length", "foreign-issuer", "dead-issuer", "consumed-bytes", "wrong-lifetime", "modified-flags"]) +def test_nonmatching_or_incomplete_dispatch_evidence_cannot_count_as_mutation_detection(mutation: str) -> None: + observation = _queued_dispatch() + queued = observation["queued_grant"] + if mutation == "digest": + queued["message_sha256"] = "f" * 64 + elif mutation == "length": + queued["bytes_observed"] += 1 + elif mutation == "foreign-issuer": + queued["peer_credentials"] = {"pid": 999, "uid": 0, "gid": 0} + elif mutation == "dead-issuer": + queued["issuer_alive"] = False + elif mutation == "consumed-bytes": + queued["bytes_still_queued"] = False + elif mutation == "wrong-lifetime": + queued["issuer_start_token"] = "987" + else: + queued["socket_flags_after"] += 1 + with pytest.raises(AssertionError) as caught: + control.assert_post_receipt_refusal(observation) + assert type(caught.value) is not control.PostReceiptGrantDispatched diff --git a/tests/test_management_recovery_epoch_native_peer.py b/tests/test_management_recovery_epoch_native_peer.py index ad94efd90..b2d568b6b 100644 --- a/tests/test_management_recovery_epoch_native_peer.py +++ b/tests/test_management_recovery_epoch_native_peer.py @@ -18,6 +18,8 @@ import pytest from scripts.recovery_epoch_native_peer import ( LATE_MUTATION_SCENARIOS, + MUTATION_SCENARIOS, + POST_RECEIPT_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, KernelPeerMessageBarrier, _container_ids, @@ -28,10 +30,11 @@ _Iovec, _local_socket_peer, _MessageHeader, + _peek_pending_frame, _peek_queued_packet, _stopped_receiver, _verify_target_loss, - mutate_after_intent, + mutate_after_publication, ) from auto_coder.management.original_attempt_evidence import OriginalFile, OriginalGeneration, OriginalLineageEvidence @@ -117,13 +120,15 @@ def test_actual_message_reader_refuses_a_return_larger_than_receiver_capacity(tm @pytest.mark.parametrize("scenario", ["", "late", "late-arbitrary-file", "late-../epoch", "after-intent-head"]) def test_late_mutation_selector_refuses_unreviewed_actions_before_access(scenario: str) -> None: with pytest.raises(ValueError, match="reviewed late mutation"): - with mutate_after_intent(None, scenario, os.getpid()): + with mutate_after_publication(None, scenario, os.getpid()): pytest.fail("an unreviewed mutation was admitted") def test_native_case_sets_are_explicit_and_distinct() -> None: assert len(REQUIRED_PEER_CASES) == 4 assert len(LATE_MUTATION_SCENARIOS) == 11 and len(set(LATE_MUTATION_SCENARIOS)) == 11 + assert POST_RECEIPT_MUTATION_SCENARIOS == ("post-receipt-ready-mutation",) + assert len(MUTATION_SCENARIOS) == 12 and not set(LATE_MUTATION_SCENARIOS).intersection(POST_RECEIPT_MUTATION_SCENARIOS) assert not REQUIRED_PEER_CASES.intersection(LATE_MUTATION_SCENARIOS) @@ -413,3 +418,23 @@ def test_actual_socket_peer_reader_does_not_accept_datagram_transport() -> None: held.enter_context(sender) held.enter_context(receiver) assert _local_socket_peer(receiver.fileno()) is None + + +@pytest.mark.parametrize("blocking", [True, False]) +def test_empty_or_queued_post_receipt_observation_never_consumes_or_changes_flags(blocking: bool) -> None: + import fcntl + + with ExitStack() as held: + sender, receiver = socket.socketpair() + held.enter_context(sender) + held.enter_context(receiver) + receiver.setblocking(blocking) + flags = fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) + assert _peek_pending_frame(receiver.fileno()) == b"" + sender.sendall(b"actual native grant bytes\n") + assert _peek_pending_frame(receiver.fileno()) == b"actual native grant bytes\n" + assert fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) == flags + assert receiver.recv(1024, socket.MSG_DONTWAIT) == b"actual native grant bytes\n" + sender.close() + assert _peek_pending_frame(receiver.fileno()) == b"" + assert fcntl.fcntl(receiver.fileno(), fcntl.F_GETFL) == flags From bab62815b56b83992ffe312d02d7b2c4dd0dd003 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 02:25:43 +0900 Subject: [PATCH 09/24] fix(management): release completed SQLite observations --- .../management-fresh-initialization.md | 4 + .../test_management_recovery_epoch_host.py | 42 +++ .../management/fresh_host_observation.py | 5 +- ...t_management_archived_cutover_container.py | 16 +- ...est_management_cutover_sqlite_lifetimes.py | 242 ++++++++++++++++++ tests/test_management_recovery_epoch_host.py | 78 ++++++ 6 files changed, 378 insertions(+), 9 deletions(-) create mode 100644 tests/test_management_cutover_sqlite_lifetimes.py diff --git a/docs/client-features/management-fresh-initialization.md b/docs/client-features/management-fresh-initialization.md index e79ae6cf2..8679d64e6 100644 --- a/docs/client-features/management-fresh-initialization.md +++ b/docs/client-features/management-fresh-initialization.md @@ -30,6 +30,10 @@ container profiles with the expected mounts are accepted. The evidence file is created once in the new owner-only guard directory. Access errors are refusal, including inability to inspect an old process namespace. +Supervisor-ledger observations close their read-only SQLite connections after +each read, including refusals. They retain no database handle beyond that check; +the owner fences and namespace descriptors have their separate required lifetimes. + The host commands accept only explicit, private files and exact container IDs: ```sh diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 43fabdf73..3ca62b1eb 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -199,6 +199,47 @@ def _command_failure(error: BaseException) -> dict[str, object] | None: return {"type": type(error).__name__, "returncode": getattr(error, "returncode", None), "diagnostics": lines[-60:]} +def _original_sidecar_diagnostics(root: Path, generations: tuple[Generation, ...]) -> list[dict[str, object]]: + """Inspect only fixed original sidecar metadata, without opening a database.""" + observations: list[dict[str, object]] = [] + for label in ("A", "B", "C", "D"): + selected = [generation for generation in generations if generation.name == label] + if not selected: + continue + for directory, database in (("state", "management-child.sqlite3"), ("journal", "management-operations.sqlite3")): + for suffix in ("-wal", "-shm", "-journal"): + row: dict[str, object] = {"generation": label, "database": database, "suffix": suffix} + try: + if len(selected) != 1 or selected[0].root != root / label or selected[0].root.resolve(strict=True) != root / label: + raise ValueError("the selected diagnostic generation has no exact original root") + parent = selected[0].path(directory) + if not stat.S_ISDIR(parent.lstat().st_mode): + raise ValueError("the selected diagnostic directory is not an original directory") + info = (parent / (database + suffix)).lstat() + except FileNotFoundError: + row["status"] = "absent" + except (OSError, ValueError) as error: + row.update(status="unavailable", error_type=type(error).__name__) + else: + kind = "regular" if stat.S_ISREG(info.st_mode) else "symlink" if stat.S_ISLNK(info.st_mode) else "directory" if stat.S_ISDIR(info.st_mode) else "socket" if stat.S_ISSOCK(info.st_mode) else "other" + row.update(status="observed", type=kind, size=info.st_size, device=info.st_dev, inode=info.st_ino) + observations.append(row) + return observations + + +def _record_failure_sidecars(result: dict[str, object], chain: Chain | None) -> None: + """Keep diagnostic failures from replacing the original native failure.""" + if chain is None: + return + try: + rows = _original_sidecar_diagnostics(chain.root, tuple(chain.generations)) + result["original_database_sidecars"] = rows + for row in rows: + logger.error("Native original database sidecar metadata: {}", canonical(row).decode()) + except Exception as error: + result["original_database_sidecars"] = [{"status": "unavailable", "error_type": type(error).__name__}] + + def _checkpoint_stopped(generation: Generation) -> None: """The stopped fixture producer settles its own stores before inspection.""" for directory in (generation.path("state"), generation.path("journal")): @@ -1147,6 +1188,7 @@ def preservation() -> object: except BaseException as error: result["status"] = "failed" result["failure_type"] = type(error).__name__ + _record_failure_sidecars(result, chain) command_failure = _command_failure(error) if command_failure is not None: result["command_failure"] = command_failure diff --git a/src/auto_coder/management/fresh_host_observation.py b/src/auto_coder/management/fresh_host_observation.py index 51a3996a8..58376e902 100644 --- a/src/auto_coder/management/fresh_host_observation.py +++ b/src/auto_coder/management/fresh_host_observation.py @@ -18,6 +18,7 @@ import subprocess import time import uuid +from contextlib import closing from dataclasses import asdict, dataclass, replace from pathlib import Path from typing import Callable @@ -313,7 +314,7 @@ def _capture_supervisor_identity(prior: PriorContainer, values: list[object], ob ledger = Path(state_sources[0]) / "management-child.sqlite3" if ledger.is_symlink() or not ledger.is_file(): raise ValueError("prior supervisor ledger unavailable") - with sqlite3.connect(ledger.as_uri() + "?mode=ro", uri=True, timeout=2) as connection: + with closing(sqlite3.connect(ledger.as_uri() + "?mode=ro", uri=True, timeout=2)) as connection: rows = connection.execute("SELECT instance_id,incarnation,pid,start_token FROM supervisor_presence").fetchall() if len(rows) != 1 or rows[0][0] != instance_id or rows[0][2:] != (1, token) or not isinstance(rows[0][1], str) or str(uuid.UUID(rows[0][1])) != rows[0][1]: raise ValueError("prior live supervisor identity differs") @@ -872,7 +873,7 @@ def assert_replacement_owner(intent: FreshInitializationIntent, incarnation: str ledger = Path(intent.host_state_root) / "management-child.sqlite3" if ledger.is_symlink() or not ledger.is_file(): raise ValueError("replacement ledger unavailable") - with sqlite3.connect(ledger.as_uri() + "?mode=ro", uri=True, timeout=2) as connection: + with closing(sqlite3.connect(ledger.as_uri() + "?mode=ro", uri=True, timeout=2)) as connection: rows = connection.execute("SELECT instance_id,incarnation,pid,start_token FROM supervisor_presence").fetchall() fields = (Path("/proc") / str(peer_pid) / "stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split() if len(fields) < 20 or rows != [(intent.new_instance, incarnation, 1, fields[19])]: diff --git a/tests/test_management_archived_cutover_container.py b/tests/test_management_archived_cutover_container.py index 5c5203bf0..ea569f16e 100644 --- a/tests/test_management_archived_cutover_container.py +++ b/tests/test_management_archived_cutover_container.py @@ -25,7 +25,7 @@ import urllib.error import urllib.request import uuid -from contextlib import ExitStack +from contextlib import ExitStack, closing from dataclasses import asdict, dataclass, replace from pathlib import Path @@ -243,11 +243,13 @@ def docker(*args: str, timeout: int = 60) -> str: @staticmethod def checkpoint(database: Path) -> None: - with sqlite3.connect(database) as connection: + # Checkpoint an existing producer store, then release its WAL/SHM + # handles before the immutable original-evidence baseline is taken. + with closing(sqlite3.connect(database.as_uri() + "?mode=rw", uri=True)) as connection: assert connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0] == 0 def grants(self): - with sqlite3.connect(self.oauth / "oauth.sqlite3") as connection: + with closing(sqlite3.connect(self.oauth / "oauth.sqlite3")) as connection: return {name: connection.execute(f"SELECT * FROM {name} ORDER BY 1").fetchall() for name in ("authority", "grants", "access_tokens", "refresh_tokens")} def prepare(self, name: str, prior: Generation | None = None, *, omit_archive: bool = False) -> Generation: @@ -567,7 +569,7 @@ def ready(self, generation: Generation, *, fresh: bool) -> None: @staticmethod def child(generation: Generation): - with sqlite3.connect((generation.path("state") / "management-child.sqlite3").as_uri() + "?mode=ro", uri=True) as connection: + with closing(sqlite3.connect((generation.path("state") / "management-child.sqlite3").as_uri() + "?mode=ro", uri=True)) as connection: return json.loads(connection.execute("SELECT record_json FROM child_state").fetchone()[0]) def prove_child(self, generation: Generation) -> None: @@ -831,7 +833,7 @@ def test_real_second_cutover_preserves_all_owner_and_key_exclusions(chain: Chain deadline = time.monotonic() + 30 while True: try: - with sqlite3.connect((a.path("state") / "management-child.sqlite3").as_uri() + "?mode=ro", uri=True) as db: + with closing(sqlite3.connect((a.path("state") / "management-child.sqlite3").as_uri() + "?mode=ro", uri=True)) as db: row = db.execute("SELECT record_json FROM child_state").fetchone() record = json.loads(row[0]) if row else {} if record.get("phase") == "running" and record.get("pid", 0) > 0: @@ -891,7 +893,7 @@ def test_real_second_cutover_preserves_all_owner_and_key_exclusions(chain: Chain assert not (c.path("state") / "fresh-launch-claimed.json").exists() ledger = c.path("state") / "management-child.sqlite3" if ledger.exists(): - with sqlite3.connect(ledger) as db: + with closing(sqlite3.connect(ledger)) as db: assert db.execute("SELECT count(*) FROM effects").fetchone()[0] == 0 assert db.execute("SELECT count(*) FROM child_state").fetchone()[0] == 0 assert not list(c.path("journal").iterdir()) @@ -908,7 +910,7 @@ def test_real_second_cutover_preserves_all_owner_and_key_exclusions(chain: Chain assert chain.call(c, "auto_coder_operation_get", {"operation_id": old.operation_id})["status"] == "NOT_FOUND" else: assert response["operation_id"] == old.operation_id and response["history_origin"] == "retained_prior_instance" - with sqlite3.connect(c.path("journal") / "management-operations.sqlite3") as db: + with closing(sqlite3.connect(c.path("journal") / "management-operations.sqlite3")) as db: assert db.execute("SELECT count(*) FROM operations").fetchone()[0] == 0 # Preserve host fences across the deliberate clean C reconstruction. held = [] diff --git a/tests/test_management_cutover_sqlite_lifetimes.py b/tests/test_management_cutover_sqlite_lifetimes.py new file mode 100644 index 000000000..647a439ae --- /dev/null +++ b/tests/test_management_cutover_sqlite_lifetimes.py @@ -0,0 +1,242 @@ +"""Real SQLite lifetimes; supplied Docker metadata is component input only.""" + +from __future__ import annotations + +import fcntl +import gc +import hashlib +import json +import os +import sqlite3 +import stat +import uuid +from contextlib import closing, contextmanager +from dataclasses import replace +from pathlib import Path + +import pytest + +from auto_coder.management import fresh_host_observation as observation +from auto_coder.management.quarantine_history import QuarantineHistory +from tests.test_management_archived_cutover_container import Chain, Generation +from tests.test_management_fresh_initialization import _intent +from tests.test_management_quarantine_history import AUTHORITY, _store + + +@contextmanager +def _without_collection(): + """Expose leaked connection cycles without relying on finalizer timing.""" + enabled = gc.isenabled() + gc.disable() + try: + yield + finally: + if enabled: + gc.enable() + + +def _open_database_handles(database: Path) -> tuple[str, ...]: + selected = {str(database), *(str(database) + suffix for suffix in ("-wal", "-shm", "-journal"))} + result = [] + for descriptor in Path("/proc/self/fd").iterdir(): + try: + target = os.readlink(descriptor) + except FileNotFoundError: + continue + if target.removesuffix(" (deleted)") in selected: + result.append(target) + return tuple(sorted(result)) + + +def _assert_no_sidecars(database: Path) -> None: + assert not _open_database_handles(database) + for suffix in ("-wal", "-shm", "-journal"): + assert not Path(str(database) + suffix).exists() + + +def test_checkpoint_closes_real_sqlite_before_quarantine_inspection(tmp_path: Path) -> None: + journal, old = _store(tmp_path / "original", "original-A", management=True) + before = journal.database.read_bytes() + identity = journal.database.stat() + _assert_no_sidecars(journal.database) + with _without_collection(): + Chain.checkpoint(journal.database) + _assert_no_sidecars(journal.database) + current = journal.database.stat() + assert (current.st_dev, current.st_ino, current.st_uid, stat.S_IMODE(current.st_mode)) == (identity.st_dev, identity.st_ino, identity.st_uid, 0o600) + assert journal.database.read_bytes() == before + with QuarantineHistory((journal,)) as history: + report = history.inspect() + assert (report.inspected_journals, report.historical_keys, report.unresolved_targets) == (1, 1, 1) + assert history.decide(old, AUTHORITY).status == "key_excluded" + assert report.launch_authority is False + _assert_no_sidecars(journal.database) + assert journal.database.read_bytes() == before + + +@pytest.mark.parametrize("reader", ["child", "grants"]) +@pytest.mark.parametrize("missing_table", [False, True]) +def test_fixture_reads_close_connections_on_success_and_error(tmp_path: Path, reader: str, missing_table: bool) -> None: + root = tmp_path / reader + root.mkdir(mode=0o700) + database = root / ("management-child.sqlite3" if reader == "child" else "oauth.sqlite3") + with closing(sqlite3.connect(database)) as producer: + assert producer.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) + if reader == "child": + producer.execute("CREATE TABLE " + ("unrelated" if missing_table else "child_state") + " (record_json TEXT)") + producer.execute("INSERT INTO " + ("unrelated" if missing_table else "child_state") + " VALUES (?)", (json.dumps({"phase": "running", "pid": 17}),)) + else: + for table in ("authority", "grants", "access_tokens", "refresh_tokens"): + producer.execute("CREATE TABLE " + ("unrelated" if missing_table and table == "grants" else table) + " (key TEXT)") + producer.commit() + database.chmod(0o600) + before = database.read_bytes() + _assert_no_sidecars(database) + if reader == "child": + generation_root = tmp_path / "generation" + generation_root.mkdir(mode=0o700) + root.rename(generation_root / "state") + database = generation_root / "state" / database.name + generation = Generation("original-A", generation_root, "a" * 64, []) + + def read(): + return Chain.child(generation) + + else: + chain = Chain.__new__(Chain) + chain.oauth = root + read = chain.grants + with _without_collection(): + if missing_table: + with pytest.raises(sqlite3.OperationalError, match="no such table"): + read() + else: + expected = {"phase": "running", "pid": 17} if reader == "child" else {name: [] for name in ("authority", "grants", "access_tokens", "refresh_tokens")} + assert read() == expected + assert not _open_database_handles(database) + assert database.read_bytes() == before + Chain.checkpoint(database) + _assert_no_sidecars(database) + assert database.read_bytes() == before + + +def test_checkpoint_refuses_a_missing_original_without_creating_it(tmp_path: Path) -> None: + database = tmp_path / "missing-original.sqlite3" + with pytest.raises(sqlite3.OperationalError, match="unable to open database file"): + Chain.checkpoint(database) + assert list(tmp_path.iterdir()) == [] + + +def test_failed_checkpoint_closes_the_original_connection(tmp_path: Path) -> None: + database = tmp_path / "corrupt-original.sqlite3" + database.write_bytes(b"retained invalid original database") + database.chmod(0o600) + with _without_collection(): + with pytest.raises(sqlite3.DatabaseError, match="file is not a database"): + Chain.checkpoint(database) + _assert_no_sidecars(database) + assert database.read_bytes() == b"retained invalid original database" + + +@pytest.mark.parametrize("reader", ["capture", "replacement"]) +@pytest.mark.parametrize("missing_table", [False, True]) +def test_actual_host_presence_reader_releases_sqlite_on_success_and_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, reader: str, missing_table: bool) -> None: + """Only Docker inspection is supplied; files, process, locks and SQLite are real.""" + intent = replace(_intent(tmp_path), state_root="/management-child-state", journal_root="/management-operations", oauth_root="/management-oauth") + config_root = tmp_path / "bootstrap" + config_root.mkdir(mode=0o700) + config = config_root / "config.json" + config.write_text( + json.dumps( + { + "instance_id": intent.new_instance, + "state_root": intent.state_root, + "journal_root": intent.journal_root, + "oauth_root": intent.oauth_root, + "listener_url": "http://127.0.0.1:8081/mcp", + "initial_runtime": "bootstrap", + "runtimes": [{"reference": "bootstrap", "receipt": "sha256:" + "b" * 64, "executable": "/usr/bin/python3", "arguments": []}], + } + ) + ) + config.chmod(0o600) + lock = tmp_path / "prior-owner.lock" + lock.write_bytes(b"") + lock.chmod(0o600) + intent = replace(intent, bootstrap_config_sha256=hashlib.sha256(config.read_bytes()).hexdigest(), owner_lock_device=lock.stat().st_dev, owner_lock_inode=lock.stat().st_ino) + pid = os.getpid() + token = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[19] + incarnation = str(uuid.uuid4()) + ledger = Path(intent.host_state_root) / "management-child.sqlite3" + with closing(sqlite3.connect(ledger)) as producer: + assert producer.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) + table = "unrelated" if missing_table else "supervisor_presence" + producer.execute(f"CREATE TABLE {table}(instance_id TEXT, incarnation TEXT, pid INTEGER, start_token TEXT)") + producer.execute(f"INSERT INTO {table} VALUES (?, ?, ?, ?)", (intent.new_instance, incarnation, 1, token)) + producer.commit() + ledger.chmod(0o600) + mounts = [ + (intent.host_state_root, intent.state_root, True), + (intent.host_journal_root, intent.journal_root, True), + (intent.host_oauth_root, intent.oauth_root, True), + (str(config_root), "/bootstrap", False), + (str(tmp_path / "guard"), "/management-initialization", False), + (str(lock), "/management-prior-owner.lock", False), + ] + values = [ + intent.new_container_id, + intent.image_digest, + True, + pid, + "running", + False, + "", + None, + None, + "bridge", + None, + [{"Type": "bind", "Source": source, "Destination": destination, "RW": writable} for source, destination, writable in mounts], + True, + "original-start", + 0, + ["python"], + [], + {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(intent.host_listener_port)}]}, + ] + inspected = [] + + def selected_inspection(container_id): + assert container_id == intent.new_container_id + inspected.append(container_id) + return values + + monkeypatch.setattr(observation, "_inspect", selected_inspection) + before = ledger.read_bytes() + identity = ledger.stat() + with lock.open("rb") as fence, _without_collection(): + fcntl.flock(fence, fcntl.LOCK_SH | fcntl.LOCK_NB) + + def read(): + if reader == "replacement": + return observation.assert_replacement_owner(intent, incarnation, pid, fence.fileno(), tmp_path / "guard") + prior = observation.PriorContainer("fixture-engine", intent.new_container_id, intent.image_digest, "original-start", 0, pid, token, "pid:[1]", "net:[2]", "bridge", "a" * 64, "/fixture", 1, 1, (observation.PriorTask(pid, token),)) + return observation._capture_supervisor_identity(prior, values, tuple(mounts), config) + + if missing_table: + with pytest.raises((sqlite3.OperationalError, observation.HostObservationUnavailable)) as error: + read() + cause = error.value if reader == "capture" else error.value.__cause__ + assert isinstance(cause, sqlite3.OperationalError) and "no such table" in str(cause) + else: + result = read() + if reader == "capture": + assert result.instance_id == intent.new_instance and result.pid == pid and result.start_token == token + else: + assert result is None and len(inspected) == 2 + assert not _open_database_handles(ledger) + after = ledger.stat() + assert (after.st_dev, after.st_ino, after.st_uid, stat.S_IMODE(after.st_mode)) == (identity.st_dev, identity.st_ino, identity.st_uid, 0o600) + assert ledger.read_bytes() == before + Chain.checkpoint(ledger) + _assert_no_sidecars(ledger) + assert ledger.read_bytes() == before diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index a55272249..ff209ef97 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -109,6 +109,84 @@ def test_command_failure_exports_bounded_diagnostics_without_inputs_or_credentia assert fixture._command_failure(ValueError("unrelated error")) is None +def _diagnostic_generation(root: Path, name: str = "A") -> Generation: + generation = Generation(name, root / name, "component-container", []) + generation.root.mkdir(parents=True) + for directory in ("state", "journal"): + generation.path(directory).mkdir() + return generation + + +def test_failure_sidecar_metadata_identifies_nonempty_shm_without_reading_stores(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + generation = _diagnostic_generation(tmp_path) + selected = generation.path("journal") / "management-operations.sqlite3-shm" + selected.write_bytes(b"private SQL and OAuth data must never be exported") + (generation.path("journal") / "management-operations.sqlite3-wal").touch() + (generation.path("state") / "oauth.sqlite3-shm").write_bytes(b"unrelated database") + (generation.path("state") / "management-child.sqlite3").write_bytes(b"private original database") + before = selected.stat() + with monkeypatch.context() as bounded: + bounded.setattr(Path, "read_bytes", lambda _: pytest.fail("sidecar diagnostics must not read file contents")) + bounded.setattr(fixture.sqlite3, "connect", lambda *_args, **_kwargs: pytest.fail("sidecar diagnostics must not connect to SQLite")) + rows = fixture._original_sidecar_diagnostics(tmp_path, (generation,)) + assert len(rows) == 6 + shm = next(row for row in rows if row["database"] == "management-operations.sqlite3" and row["suffix"] == "-shm") + assert shm == {"generation": "A", "database": "management-operations.sqlite3", "suffix": "-shm", "status": "observed", "type": "regular", "size": before.st_size, "device": before.st_dev, "inode": before.st_ino} + wal = next(row for row in rows if row["database"] == "management-operations.sqlite3" and row["suffix"] == "-wal") + assert wal["status"] == "observed" and wal["size"] == 0 + assert sum(row["status"] == "absent" for row in rows) == 4 + assert selected.stat() == before + assert "private" not in json.dumps(rows) and "oauth" not in json.dumps(rows) and str(tmp_path) not in json.dumps(rows) + + +def test_failure_sidecar_diagnostics_do_not_follow_links_or_export_other_scopes(tmp_path: Path) -> None: + generation = _diagnostic_generation(tmp_path) + private = tmp_path / "private-target" + private.write_bytes(b"not inspected") + sidecar = generation.path("state") / "management-child.sqlite3-shm" + sidecar.symlink_to(private) + (generation.path("journal") / "management-operations.sqlite3-journal").mkdir() + unrelated = _diagnostic_generation(tmp_path, "recovery-target") + rows = fixture._original_sidecar_diagnostics(tmp_path, (generation, unrelated)) + assert len(rows) == 6 and {row["generation"] for row in rows} == {"A"} + linked = next(row for row in rows if row.get("type") == "symlink") + assert linked["inode"] == sidecar.lstat().st_ino and linked["inode"] != private.stat().st_ino + assert any(row.get("type") == "directory" for row in rows) + assert "private-target" not in json.dumps(rows) + + +def test_failure_sidecar_diagnostics_are_finite_and_reject_ambiguous_original_roots(tmp_path: Path) -> None: + generations = tuple(_diagnostic_generation(tmp_path, name) for name in ("A", "B", "C", "D")) + assert len(fixture._original_sidecar_diagnostics(tmp_path, generations)) == 24 + ambiguous = fixture._original_sidecar_diagnostics(tmp_path, generations + (generations[0],) * 20) + assert len(ambiguous) == 24 + assert all(row["status"] == "unavailable" and row["error_type"] == "ValueError" for row in ambiguous if row["generation"] == "A") + foreign = Generation("A", tmp_path / "not-original-A", "component-container", []) + assert all(row["status"] == "unavailable" for row in fixture._original_sidecar_diagnostics(tmp_path, (foreign,))) + + +def test_unavailable_sidecar_diagnostics_preserve_the_original_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + chain = object.__new__(fixture.Chain) + chain.root = tmp_path + chain.generations = [_diagnostic_generation(tmp_path)] + result = {"status": "failed", "failure_type": "AssertionError"} + + def unavailable(*_args): + raise PermissionError("private diagnostic input must not be exported") + + monkeypatch.setattr(fixture, "_original_sidecar_diagnostics", unavailable) + original = AssertionError("the original native oracle failed") + with pytest.raises(AssertionError) as caught: + try: + raise original + except AssertionError: + fixture._record_failure_sidecars(result, chain) + raise + assert caught.value is original + assert result["original_database_sidecars"] == [{"status": "unavailable", "error_type": "PermissionError"}] + assert "private" not in json.dumps(result) + + def test_no_child_or_admission_oracle_rejects_any_native_store(tmp_path: Path) -> None: state, journal = tmp_path / "state", tmp_path / "journal" state.mkdir() From 080bd7c02676db9720d19a05ad16281c2b198cee Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 02:46:24 +0900 Subject: [PATCH 10/24] fix(management): finalize native fixture producers before recovery --- scripts/recovery_epoch_native_negatives.py | 3 +- .../test_management_recovery_epoch_host.py | 58 ++++++- ...t_management_containment_route_manifest.py | 15 ++ ...anagement_recovery_fixture_finalization.py | 161 ++++++++++++++++++ 4 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 tests/test_management_recovery_fixture_finalization.py diff --git a/scripts/recovery_epoch_native_negatives.py b/scripts/recovery_epoch_native_negatives.py index 3327a9aef..5f4f62db3 100644 --- a/scripts/recovery_epoch_native_negatives.py +++ b/scripts/recovery_epoch_native_negatives.py @@ -565,7 +565,8 @@ def _unmasked_restarter(context: NativeNegativeContext) -> Iterator[None]: identity = _projection(path) try: _systemctl("daemon-reload") - actual = _systemd_unit(name) + # Match the enrolled container identity, as the production route scan does. + actual = _systemd_unit(name, extra_cues=(context.enrollment.originals[0].management_id,)) if actual.inhibited or not actual.management_candidate or actual.main_pid != 0 or actual.active_state != "inactive" or name not in _systemd_catalog(): raise AssertionError("the actual installed restarter is not the intended unmasked inactive route") yield diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 3ca62b1eb..6d00a1020 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -43,7 +43,7 @@ from auto_coder.management.operation_store import ManagementOperationStore from auto_coder.management.operations import ActionParameter, ManagedTarget, OperationIntent, OperationPhase from auto_coder.management.original_attempt_evidence import derive_lineage, observe, verify_lineage -from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, source_identity +from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, read_original, source_identity from auto_coder.management.recovery_containment_model import RecoveryWatcher from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, AUTHORIZATION_ROOT, AUTHORIZATION_SCHEMA, PEER_ROOT, PEER_SOCKET, EpochAuthorization, validate_uv_version from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA, ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource @@ -247,6 +247,61 @@ def _checkpoint_stopped(generation: Generation) -> None: Chain.checkpoint(database) +def _finalize_original_stores(chain: Chain, captures: tuple[CapturedGeneration, ...]) -> None: + """Finish known stopped fixture producers before any recovery baseline.""" + if set(os.listdir(EPOCH_ROOT)) != {"epoch.lock"} or AUTHORIZATION_FILE.exists() or AUTHORIZATION_FILE.is_symlink() or (chain.root / "target").exists(): + raise AssertionError("original fixture finalization is only permitted before recovery preparation") + if chain.root != chain.root.resolve(strict=True) or len(captures) != 3 or {captured.original.generation_id for captured in captures} != {"A", "B", "C"}: + raise AssertionError("only the three explicitly captured original generations may be finalized") + if any(process.poll() != 0 for process in chain.processes): + raise AssertionError("an original cutover reader has not completed before finalization") + originals: list[tuple[Path, bytes, os.stat_result]] = [] + for captured in captures: + original = captured.original + selected = [generation for generation in chain.generations if generation.name == original.generation_id] + if len(selected) != 1 or original.state != "started" or original.instance_id != original.generation_id: + raise AssertionError("the selected original fixture generation is missing or ambiguous") + generation = selected[0] + if generation.root != chain.root / original.generation_id or (generation.container, generation.proxy) != (original.management_id, original.proxy_id): + raise AssertionError("the selected original fixture differs from its captured Docker objects") + for identifier in (original.management_id, original.proxy_id): + state = json.loads(chain.docker("inspect", "--format", "{{json .State}}", identifier)) + if state.get("Status") != "exited" or state.get("Running") is not False or state.get("Pid") != 0 or state.get("Restarting") is not False: + raise AssertionError("an original manager or proxy is not concretely stopped before finalization") + for directory, name in (("state", "management-child.sqlite3"), ("journal", "management-operations.sqlite3")): + parent = generation.path(directory) + known = {name, *(name + suffix for suffix in ("-wal", "-shm", "-journal"))} + if not stat.S_ISDIR(parent.lstat().st_mode) or {path.name for path in parent.glob("*.sqlite3*")} - known: + raise AssertionError("original fixture finalization found an unknown store or directory") + database = parent / name + encoded, _ = read_original(database) + before = database.lstat() + for suffix in ("-wal", "-shm", "-journal"): + sidecar = database.with_name(name + suffix) + try: + info = sidecar.lstat() + except FileNotFoundError: + continue + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1: + raise AssertionError("an original fixture sidecar is not its private regular object") + if suffix != "-shm" and info.st_size: + raise AssertionError("an original fixture still has nonempty transaction state") + originals.append((database, encoded, before)) + # A later archived read may have recreated SQLite's derived SHM index + # after an earlier producer checkpoint. Every actual reader is now gone; + # the existing checkpoint may close that index, never apply new records. + for database, encoded, before in originals: + Chain.checkpoint(database) + current, _ = read_original(database) + after = database.lstat() + fields = ("st_dev", "st_ino", "st_ctime_ns", "st_mtime_ns", "st_mode", "st_uid", "st_gid", "st_nlink", "st_size") + if current != encoded or any(getattr(before, field) != getattr(after, field) for field in fields): + raise AssertionError("finalizing a stopped fixture changed original database bytes or identity") + if any(database.with_name(database.name + suffix).exists() or database.with_name(database.name + suffix).is_symlink() for suffix in ("-wal", "-shm", "-journal")): + raise AssertionError("original fixture sidecars remain after all tracked readers completed") + logger.info("Finalized six unchanged original A/B/C stores after all cutover readers stopped, before recovery preparation") + + def _no_target_child(target: Generation, *, running: bool) -> None: assert not set(os.listdir(target.path("state"))) - {".supervisor-owner.lock", "logs"}, "prelaunch target created native consumer state" assert not os.listdir(target.path("journal")), "prelaunch target admitted an operation" @@ -1042,6 +1097,7 @@ def run_fixture(*, authorization_id: str, source_repository: Path, source_token_ watchers.append(watcher) chain.stop(third) _checkpoint_stopped(third) + _finalize_original_stores(chain, tuple(captures)) for captured in captures: assert captured.original.namespace_pin is not None assert os.readlink(f"/proc/self/fd/{captured.namespace_fd}") == captured.original.namespace_pin.namespace diff --git a/tests/test_management_containment_route_manifest.py b/tests/test_management_containment_route_manifest.py index 789ab74ae..5d17a1f1c 100644 --- a/tests/test_management_containment_route_manifest.py +++ b/tests/test_management_containment_route_manifest.py @@ -39,6 +39,21 @@ def test_systemd_timer_without_service_main_pid_is_fingerprinted(monkeypatch: py assert fact.inhibited is False +def test_generic_docker_restarter_uses_the_enrolled_container_identity(monkeypatch: pytest.MonkeyPatch) -> None: + unit = "auto-coder-recovery-negative-" + "1" * 32 + ".service" + selected = "2" * 64 + shown = f"Id={unit}\nLoadState=loaded\nUnitFileState=static\nActiveState=inactive\nSubState=dead\nFragmentPath=/run/systemd/system/{unit}\nNeedDaemonReload=no\nMainPID=0\nExecStart=/usr/bin/docker start {selected}\n" + source = f"[Service]\nType=oneshot\nExecStart=/usr/bin/docker start {selected}\n" + monkeypatch.setattr(routes, "_command", lambda *args, **kwargs: shown if args[1] == "show" else source) + assert not routes._systemd_unit(unit).management_candidate + assert not routes._systemd_unit(unit, extra_cues=("3" * 64,)).management_candidate + fact = routes._systemd_unit(unit, extra_cues=(selected,)) + assert fact.management_candidate + assert not fact.inhibited + assert fact.main_pid == 0 + assert fact.active_state == "inactive" + + @pytest.mark.parametrize("unit", ("apport-autoreport.path", "multi-user.target", "etc-fstab.mount", "proc-sys-fs-binfmt_misc.automount", "dev-zram0.swap")) def test_other_service_activators_are_fingerprinted_without_main_pid(monkeypatch: pytest.MonkeyPatch, unit: str) -> None: shown = "Id=" + unit + "\nLoadState=loaded\nUnitFileState=enabled\nActiveState=active\nSubState=running\nFragmentPath=/etc/systemd/system/" + unit + "\nNeedDaemonReload=no\n" diff --git a/tests/test_management_recovery_fixture_finalization.py b/tests/test_management_recovery_fixture_finalization.py new file mode 100644 index 000000000..ccc5dc357 --- /dev/null +++ b/tests/test_management_recovery_fixture_finalization.py @@ -0,0 +1,161 @@ +"""Real original SQLite chronology with explicit component Docker state inputs. + +Only the separate native VM proves actual stopped Docker/host producers. These +regressions exercise their bounded fixture finalization against real SQLite, +RetainedJournal and QuarantineHistory without changing those production readers. +""" + +from __future__ import annotations + +import gc +import json +import sqlite3 +import subprocess +import sys +from contextlib import closing +from pathlib import Path +from types import SimpleNamespace + +import pytest +from scripts import test_management_recovery_epoch_host as fixture + +from auto_coder.management.operations import OperationPhase +from auto_coder.management.quarantine_history import QuarantineHistory, QuarantineUnavailable +from auto_coder.management.retained_journal import RetainedJournal +from tests.test_management_archived_cutover_container import Chain, Generation +from tests.test_management_quarantine_history import _store + + +@pytest.fixture +def originals(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + epoch = tmp_path / "epoch" + epoch.mkdir() + (epoch / "epoch.lock").touch() + monkeypatch.setattr(fixture, "EPOCH_ROOT", epoch) + monkeypatch.setattr(fixture, "AUTHORIZATION_FILE", tmp_path / "authorization.json") + chain = Chain.__new__(Chain) + chain.root = tmp_path / "deployment" + chain.root.mkdir() + chain.generations = [] + cutover = subprocess.Popen([sys.executable, "-I", "-B", "-c", "pass"]) + assert cutover.wait(timeout=10) == 0 + chain.processes = [cutover] + chain.docker = lambda *_args: json.dumps({"Status": "exited", "Running": False, "Pid": 0, "Restarting": False}) + journals, captures = {}, [] + for name in ("A", "B", "C"): + root = chain.root / name + root.mkdir() + journal, _ = _store(root / "journal", name, management=True, phase=OperationPhase.NEEDS_ATTENTION if name == "C" else OperationPhase.SUCCEEDED) + journals[name] = journal + state = root / "state" + state.mkdir() + ledger = state / "management-child.sqlite3" + with closing(sqlite3.connect(ledger)) as connection: + assert connection.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) + connection.execute("CREATE TABLE child_state (value TEXT)") + connection.commit() + ledger.chmod(0o600) + generation = Generation(name, root, "manager-" + name, [], proxy="proxy-" + name) + chain.generations.append(generation) + original = SimpleNamespace(generation_id=name, instance_id=name, state="started", management_id=generation.container, proxy_id=generation.proxy) + captures.append(SimpleNamespace(original=original)) + failed = Generation("D", chain.root / "D", "never-created-manager", [], proxy="never-created-proxy") + failed.root.mkdir() + (failed.root / "retained-never-created").write_bytes(b"original failed branch") + chain.generations.append(failed) + return chain, tuple(captures), journals + + +def _late_archived_read(journal) -> None: + # This earlier checkpoint precedes the genuine later archived read. + Chain.checkpoint(journal.database) + assert not Path(str(journal.database) + "-shm").exists() + with RetainedJournal(journal.database, journal.fence, database_identity=(journal.database_device, journal.database_inode), fence_identity=(journal.fence_device, journal.fence_inode), prior_management_instance="A") as reader: + assert len(reader.read()) == 1 + assert Path(str(journal.database) + "-wal").stat().st_size == 0 + assert Path(str(journal.database) + "-shm").stat().st_size > 0 + + +def _retained_bytes(chain) -> dict[str, tuple[bytes, int, int, int]]: + result = {} + for path in sorted(chain.root.rglob("*")): + if path.is_file() and not path.is_symlink(): + info = path.stat() + result[str(path)] = (path.read_bytes(), info.st_dev, info.st_ino, info.st_ctime_ns) + return result + + +def test_final_checkpoint_follows_last_archived_read_and_preserves_original_authority(originals) -> None: + chain, captures, journals = originals + collection = gc.isenabled() + gc.disable() + try: + _late_archived_read(journals["A"]) + before = _retained_bytes(chain) + with pytest.raises(QuarantineUnavailable, match="nonempty transaction sidecar"): + QuarantineHistory((journals["A"],)) + fixture._finalize_original_stores(chain, captures) + after = _retained_bytes(chain) + assert {name: value for name, value in before.items() if not name.endswith(("-wal", "-shm", "-journal"))} == after + assert (chain.root / "D" / "retained-never-created").read_bytes() == b"original failed branch" + for name, journal in journals.items(): + with QuarantineHistory((journal,)) as history: + report = history.inspect() + assert report.historical_keys == 1 + assert report.unresolved_targets == (1 if name == "C" else 0) + assert report.launch_authority is False + assert _retained_bytes(chain) == after + finally: + if collection: + gc.enable() + + +@pytest.mark.parametrize("fault", ["nonempty-wal", "hot-journal", "missing-store", "unknown-store", "linked-sidecar", "running-manager", "running-proxy", "live-cutover", "published-inventory", "installed-authorization", "target-prepared", "wrong-capture", "missing-capture"]) +def test_finalization_refuses_uncertain_scope_before_any_checkpoint(originals, fault: str) -> None: + chain, captures, journals = originals + _late_archived_read(journals["A"]) + stopped = chain.docker + if fault in {"nonempty-wal", "hot-journal"}: + path = Path(str(journals["B"].database) + ("-wal" if fault == "nonempty-wal" else "-journal")) + path.write_bytes(b"retained transaction uncertainty") + path.chmod(0o600) + elif fault == "missing-store": + journals["B"].database.unlink() + elif fault == "unknown-store": + (journals["B"].database.parent / "unknown.sqlite3").write_bytes(b"unknown original store") + elif fault == "linked-sidecar": + Path(str(journals["B"].database) + "-shm").symlink_to(journals["A"].database) + elif fault in {"running-manager", "running-proxy"}: + selected = "manager-B" if fault == "running-manager" else "proxy-B" + chain.docker = lambda *args: json.dumps({"Status": "running", "Running": True, "Pid": 123, "Restarting": False}) if args[-1] == selected else stopped(*args) + elif fault == "live-cutover": + chain.processes = [SimpleNamespace(poll=lambda: None)] + elif fault == "published-inventory": + (fixture.EPOCH_ROOT / "inventory.json").write_bytes(b"already selected recovery epoch") + elif fault == "installed-authorization": + fixture.AUTHORIZATION_FILE.write_bytes(b"already authorized recovery") + elif fault == "target-prepared": + (chain.root / "target").mkdir() + elif fault == "wrong-capture": + captures[1].original.management_id = "another-manager" + else: + captures = captures[:-1] + before = _retained_bytes(chain) + with pytest.raises((AssertionError, ValueError, FileNotFoundError)): + fixture._finalize_original_stores(chain, captures) + assert _retained_bytes(chain) == before + assert Path(str(journals["A"].database) + "-shm").stat().st_size > 0 + + +def test_finalization_rejects_any_database_change_even_if_checkpoint_returns(originals, monkeypatch: pytest.MonkeyPatch) -> None: + chain, captures, _ = originals + actual = Chain.checkpoint + + def changed(database: Path) -> None: + actual(database) + with database.open("ab") as stream: + stream.write(b"unexpected original database change") + + monkeypatch.setattr(Chain, "checkpoint", changed) + with pytest.raises(AssertionError, match="changed original database bytes or identity"): + fixture._finalize_original_stores(chain, captures) From b767f2be00081bc006793e71a25ebb601f994ce9 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 03:04:00 +0900 Subject: [PATCH 11/24] fix(management): retain the complete original database profile --- .../test_management_recovery_epoch_host.py | 9 ++--- tests/test_management_recovery_epoch_host.py | 10 +++--- ...anagement_recovery_fixture_finalization.py | 36 ++++++++++++++----- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 6d00a1020..906cef80e 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -56,6 +56,7 @@ logger = get_logger(__name__) FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") RESULT_SCHEMA = "auto-coder-native-recovery-epoch-result-v1" +ORIGINAL_FIXTURE_DATABASES = (("state", "management-child.sqlite3"), ("state", "management-self-update.sqlite3"), ("state", "management-recovery.sqlite3"), ("journal", "management-operations.sqlite3")) ORIGINAL_NEGATIVE_CASES = frozenset({"original-journal-absence-refused", "changed-installed-ready-refused", "changed-target-management-config-refused"}) LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused", "cross-deployment-original-lineage-refused"}) FACTORY_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES if case.startswith("factory-")) @@ -206,7 +207,7 @@ def _original_sidecar_diagnostics(root: Path, generations: tuple[Generation, ... selected = [generation for generation in generations if generation.name == label] if not selected: continue - for directory, database in (("state", "management-child.sqlite3"), ("journal", "management-operations.sqlite3")): + for directory, database in ORIGINAL_FIXTURE_DATABASES: for suffix in ("-wal", "-shm", "-journal"): row: dict[str, object] = {"generation": label, "database": database, "suffix": suffix} try: @@ -268,9 +269,9 @@ def _finalize_original_stores(chain: Chain, captures: tuple[CapturedGeneration, state = json.loads(chain.docker("inspect", "--format", "{{json .State}}", identifier)) if state.get("Status") != "exited" or state.get("Running") is not False or state.get("Pid") != 0 or state.get("Restarting") is not False: raise AssertionError("an original manager or proxy is not concretely stopped before finalization") - for directory, name in (("state", "management-child.sqlite3"), ("journal", "management-operations.sqlite3")): + for directory, name in ORIGINAL_FIXTURE_DATABASES: parent = generation.path(directory) - known = {name, *(name + suffix for suffix in ("-wal", "-shm", "-journal"))} + known = {database + suffix for selected_directory, database in ORIGINAL_FIXTURE_DATABASES if selected_directory == directory for suffix in ("", "-wal", "-shm", "-journal")} if not stat.S_ISDIR(parent.lstat().st_mode) or {path.name for path in parent.glob("*.sqlite3*")} - known: raise AssertionError("original fixture finalization found an unknown store or directory") database = parent / name @@ -299,7 +300,7 @@ def _finalize_original_stores(chain: Chain, captures: tuple[CapturedGeneration, raise AssertionError("finalizing a stopped fixture changed original database bytes or identity") if any(database.with_name(database.name + suffix).exists() or database.with_name(database.name + suffix).is_symlink() for suffix in ("-wal", "-shm", "-journal")): raise AssertionError("original fixture sidecars remain after all tracked readers completed") - logger.info("Finalized six unchanged original A/B/C stores after all cutover readers stopped, before recovery preparation") + logger.info("Finalized twelve unchanged original A/B/C stores after all cutover readers stopped, before recovery preparation") def _no_target_child(target: Generation, *, running: bool) -> None: diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index ff209ef97..3b57cf768 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -129,12 +129,12 @@ def test_failure_sidecar_metadata_identifies_nonempty_shm_without_reading_stores bounded.setattr(Path, "read_bytes", lambda _: pytest.fail("sidecar diagnostics must not read file contents")) bounded.setattr(fixture.sqlite3, "connect", lambda *_args, **_kwargs: pytest.fail("sidecar diagnostics must not connect to SQLite")) rows = fixture._original_sidecar_diagnostics(tmp_path, (generation,)) - assert len(rows) == 6 + assert len(rows) == 12 shm = next(row for row in rows if row["database"] == "management-operations.sqlite3" and row["suffix"] == "-shm") assert shm == {"generation": "A", "database": "management-operations.sqlite3", "suffix": "-shm", "status": "observed", "type": "regular", "size": before.st_size, "device": before.st_dev, "inode": before.st_ino} wal = next(row for row in rows if row["database"] == "management-operations.sqlite3" and row["suffix"] == "-wal") assert wal["status"] == "observed" and wal["size"] == 0 - assert sum(row["status"] == "absent" for row in rows) == 4 + assert sum(row["status"] == "absent" for row in rows) == 10 assert selected.stat() == before assert "private" not in json.dumps(rows) and "oauth" not in json.dumps(rows) and str(tmp_path) not in json.dumps(rows) @@ -148,7 +148,7 @@ def test_failure_sidecar_diagnostics_do_not_follow_links_or_export_other_scopes( (generation.path("journal") / "management-operations.sqlite3-journal").mkdir() unrelated = _diagnostic_generation(tmp_path, "recovery-target") rows = fixture._original_sidecar_diagnostics(tmp_path, (generation, unrelated)) - assert len(rows) == 6 and {row["generation"] for row in rows} == {"A"} + assert len(rows) == 12 and {row["generation"] for row in rows} == {"A"} linked = next(row for row in rows if row.get("type") == "symlink") assert linked["inode"] == sidecar.lstat().st_ino and linked["inode"] != private.stat().st_ino assert any(row.get("type") == "directory" for row in rows) @@ -157,9 +157,9 @@ def test_failure_sidecar_diagnostics_do_not_follow_links_or_export_other_scopes( def test_failure_sidecar_diagnostics_are_finite_and_reject_ambiguous_original_roots(tmp_path: Path) -> None: generations = tuple(_diagnostic_generation(tmp_path, name) for name in ("A", "B", "C", "D")) - assert len(fixture._original_sidecar_diagnostics(tmp_path, generations)) == 24 + assert len(fixture._original_sidecar_diagnostics(tmp_path, generations)) == 48 ambiguous = fixture._original_sidecar_diagnostics(tmp_path, generations + (generations[0],) * 20) - assert len(ambiguous) == 24 + assert len(ambiguous) == 48 assert all(row["status"] == "unavailable" and row["error_type"] == "ValueError" for row in ambiguous if row["generation"] == "A") foreign = Generation("A", tmp_path / "not-original-A", "component-container", []) assert all(row["status"] == "unavailable" for row in fixture._original_sidecar_diagnostics(tmp_path, (foreign,))) diff --git a/tests/test_management_recovery_fixture_finalization.py b/tests/test_management_recovery_fixture_finalization.py index ccc5dc357..ee3a70d69 100644 --- a/tests/test_management_recovery_fixture_finalization.py +++ b/tests/test_management_recovery_fixture_finalization.py @@ -9,7 +9,7 @@ import gc import json -import sqlite3 +import os import subprocess import sys from contextlib import closing @@ -19,9 +19,12 @@ import pytest from scripts import test_management_recovery_epoch_host as fixture +from auto_coder.management.bootstrap_store import BootstrapStore from auto_coder.management.operations import OperationPhase from auto_coder.management.quarantine_history import QuarantineHistory, QuarantineUnavailable +from auto_coder.management.recovery_mailbox import RecoveryMailbox from auto_coder.management.retained_journal import RetainedJournal +from auto_coder.management.self_update_handoff import HandoffStore from tests.test_management_archived_cutover_container import Chain, Generation from tests.test_management_quarantine_history import _store @@ -48,13 +51,19 @@ def originals(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): journal, _ = _store(root / "journal", name, management=True, phase=OperationPhase.NEEDS_ATTENTION if name == "C" else OperationPhase.SUCCEEDED) journals[name] = journal state = root / "state" - state.mkdir() - ledger = state / "management-child.sqlite3" - with closing(sqlite3.connect(ledger)) as connection: - assert connection.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) - connection.execute("CREATE TABLE child_state (value TEXT)") - connection.commit() - ledger.chmod(0o600) + state.mkdir(mode=0o700) + previous_umask = os.umask(0o077) + try: + with closing(BootstrapStore(state, name)._connect()): + pass + with closing(HandoffStore(state, name)._connect()): + pass + RecoveryMailbox(state, name).initialize() + finally: + os.umask(previous_umask) + assert {path.name for path in state.glob("*.sqlite3")} == {"management-child.sqlite3", "management-self-update.sqlite3", "management-recovery.sqlite3"} + for path in state.glob("*.sqlite3"): + Chain.checkpoint(path) generation = Generation(name, root, "manager-" + name, [], proxy="proxy-" + name) chain.generations.append(generation) original = SimpleNamespace(generation_id=name, instance_id=name, state="started", management_id=generation.container, proxy_id=generation.proxy) @@ -110,6 +119,17 @@ def test_final_checkpoint_follows_last_archived_read_and_preserves_original_auth gc.enable() +@pytest.mark.parametrize("name", ["management-self-update.sqlite3", "management-recovery.sqlite3"]) +def test_all_real_original_mailboxes_are_required_and_never_reconstructed(originals, name: str) -> None: + chain, captures, _ = originals + missing = chain.root / "B" / "state" / name + missing.unlink() + before = _retained_bytes(chain) + with pytest.raises(FileNotFoundError): + fixture._finalize_original_stores(chain, captures) + assert not missing.exists() and _retained_bytes(chain) == before + + @pytest.mark.parametrize("fault", ["nonempty-wal", "hot-journal", "missing-store", "unknown-store", "linked-sidecar", "running-manager", "running-proxy", "live-cutover", "published-inventory", "installed-authorization", "target-prepared", "wrong-capture", "missing-capture"]) def test_finalization_refuses_uncertain_scope_before_any_checkpoint(originals, fault: str) -> None: chain, captures, journals = originals From 7e86de9e64a25a76e1b718e99937429dd75a8307 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 03:04:49 +0900 Subject: [PATCH 12/24] fix(workspace): keep patch capture from refreshing the caller index --- src/auto_coder/worktree_utils.py | 3 ++- tests/test_worktree_isolation.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/auto_coder/worktree_utils.py b/src/auto_coder/worktree_utils.py index 9823d9c61..f38874849 100644 --- a/src/auto_coder/worktree_utils.py +++ b/src/auto_coder/worktree_utils.py @@ -494,7 +494,8 @@ def _capture_source(target: Path, workspace: Path, ownership: Optional[LocalWork def _capture_git_patch(root: Path, destination: Path, *args: str) -> Path: """Spool binary patches to disk instead of retaining their full stdout.""" with destination.open("wb") as stream: - result = subprocess.run(["git", "diff", *args, "--binary", "--full-index"], cwd=root, stdout=stream, stderr=subprocess.PIPE) + # Diff's cached-stat refresh would otherwise rewrite the bound caller index. + result = subprocess.run(["git", "-c", "diff.autoRefreshIndex=false", "diff", *args, "--binary", "--full-index"], cwd=root, stdout=stream, stderr=subprocess.PIPE) if result.returncode != 0: raise WorkspacePreparationError(f"unable to capture Git patch: {os.fsdecode(result.stderr).strip()}") return destination diff --git a/tests/test_worktree_isolation.py b/tests/test_worktree_isolation.py index 577975fbc..6ea03dc0d 100644 --- a/tests/test_worktree_isolation.py +++ b/tests/test_worktree_isolation.py @@ -517,6 +517,28 @@ def test_tracked_file_permissions_survive_preparation_and_noop_handoff(tmp_path: assert stat.S_IMODE(tracked.stat().st_mode) == 0o600 +@pytest.mark.parametrize("is_noedit", [False, True]) +def test_metadata_only_change_preserves_exact_caller_index_during_preparation(tmp_path: Path, is_noedit: bool) -> None: + repo = _init_repo(tmp_path) + tracked = repo / "tracked.txt" + observed = tracked.stat() + # Force a real cached-stat mismatch without a wall-clock race or content edit. + os.utime(tracked, ns=(observed.st_atime_ns, observed.st_mtime_ns - 2_000_000_000)) + tracked.chmod(0o600) + index = repo / ".git" / "index" + original_index = index.read_bytes() + + with isolated_local_llm_worktree(repo, is_noedit=is_noedit) as workspace: + assert index.read_bytes() == original_index + private = Path(workspace) / "tracked.txt" + assert private.read_text() == "initial content\n" + assert stat.S_IMODE(private.stat().st_mode) == 0o600 + + assert index.read_bytes() == original_index + assert tracked.read_text() == "initial content\n" + assert stat.S_IMODE(tracked.stat().st_mode) == 0o600 + + def test_disposing_one_private_workspace_preserves_peer_workspace_and_caller_refs(tmp_path: Path) -> None: repo = _init_repo(tmp_path) initial_commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() From 25d8e1978e1ce5bacfbcba8b9d39ff8e9f4e7246 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 03:19:00 +0900 Subject: [PATCH 13/24] fix(management): preserve original authenticated readback stores --- .../management-recovery-epoch.md | 9 ++ .../test_management_recovery_epoch_host.py | 91 +++++++++++++++---- tests/test_management_recovery_epoch_host.py | 45 ++++++++- ...anagement_recovery_fixture_finalization.py | 54 ++++++++++- 4 files changed, 175 insertions(+), 24 deletions(-) diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md index b4aec7c4e..aee7b54f3 100644 --- a/docs/client-features/management-recovery-epoch.md +++ b/docs/client-features/management-recovery-epoch.md @@ -217,6 +217,15 @@ succeed. A missing case, failed VM, unavailable native kernel operation or faile preservation check cannot become a passed result. The VM runner and reviewed reproduction procedure are documented in `scripts/recovery_vm/README.md`. +Before recovery preparation, the fixture proves that all original A/B/C managers, +proxies and cutover readers have stopped. It may then close only the empty WAL +and derived SHM residue of their twelve known original stores, preserving every +database's bytes and identity. The separate A/B readiness observation stores are +required and preserved without checkpointing; unresolved C has none. Unknown +stores, nonempty WAL/rollback journals, or readiness sidecars refuse. This fixture +setup does not repair an operational recovery source or change the production +quarantine policy. + | Contract | Actual fixture action and independent outcome | | --- | --- | | AS-B01 | Run genuine genesis, successful successor, never-created failed branch and unresolved successor producers. Verify full original lineage, reject forged/cross-deployment/stale reports, remove an original journal, and mutate the retained failed branch's exact terminal evidence after intent. Refusal must preserve the original uncertainty and stores. | diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 906cef80e..16049360e 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -57,6 +57,7 @@ FIXTURE_ROOT = Path("/var/lib/auto-coder/recovery-epoch-fixtures") RESULT_SCHEMA = "auto-coder-native-recovery-epoch-result-v1" ORIGINAL_FIXTURE_DATABASES = (("state", "management-child.sqlite3"), ("state", "management-self-update.sqlite3"), ("state", "management-recovery.sqlite3"), ("journal", "management-operations.sqlite3")) +READINESS_OBSERVATION_DATABASE = ("state", "management-readiness.sqlite3") ORIGINAL_NEGATIVE_CASES = frozenset({"original-journal-absence-refused", "changed-installed-ready-refused", "changed-target-management-config-refused"}) LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused", "cross-deployment-original-lineage-refused"}) FACTORY_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES if case.startswith("factory-")) @@ -200,6 +201,16 @@ def _command_failure(error: BaseException) -> dict[str, object] | None: return {"type": type(error).__name__, "returncode": getattr(error, "returncode", None), "diagnostics": lines[-60:]} +def _fixture_databases(label: str) -> tuple[tuple[str, str], ...]: + # A/B completed an authenticated readback; C deliberately did not. The + # separate readiness observation is never original lineage authority. + return ORIGINAL_FIXTURE_DATABASES + ((READINESS_OBSERVATION_DATABASE,) if label in {"A", "B"} else ()) + + +def _file_type(mode: int) -> str: + return "regular" if stat.S_ISREG(mode) else "symlink" if stat.S_ISLNK(mode) else "directory" if stat.S_ISDIR(mode) else "socket" if stat.S_ISSOCK(mode) else "other" + + def _original_sidecar_diagnostics(root: Path, generations: tuple[Generation, ...]) -> list[dict[str, object]]: """Inspect only fixed original sidecar metadata, without opening a database.""" observations: list[dict[str, object]] = [] @@ -207,7 +218,7 @@ def _original_sidecar_diagnostics(root: Path, generations: tuple[Generation, ... selected = [generation for generation in generations if generation.name == label] if not selected: continue - for directory, database in ORIGINAL_FIXTURE_DATABASES: + for directory, database in _fixture_databases(label): for suffix in ("-wal", "-shm", "-journal"): row: dict[str, object] = {"generation": label, "database": database, "suffix": suffix} try: @@ -222,23 +233,58 @@ def _original_sidecar_diagnostics(root: Path, generations: tuple[Generation, ... except (OSError, ValueError) as error: row.update(status="unavailable", error_type=type(error).__name__) else: - kind = "regular" if stat.S_ISREG(info.st_mode) else "symlink" if stat.S_ISLNK(info.st_mode) else "directory" if stat.S_ISDIR(info.st_mode) else "socket" if stat.S_ISSOCK(info.st_mode) else "other" - row.update(status="observed", type=kind, size=info.st_size, device=info.st_dev, inode=info.st_ino) + row.update(status="observed", type=_file_type(info.st_mode), size=info.st_size, device=info.st_dev, inode=info.st_ino) observations.append(row) return observations +def _original_namespace_diagnostics(root: Path, generations: tuple[Generation, ...]) -> list[dict[str, object]]: + """Name at most sixteen unexpected SQLite entries in each original root.""" + observations: list[dict[str, object]] = [] + for label in ("A", "B", "C", "D"): + selected = [generation for generation in generations if generation.name == label] + if not selected: + continue + for directory in ("state", "journal"): + row: dict[str, object] = {"generation": label, "directory": directory} + try: + if len(selected) != 1 or selected[0].root != root / label or selected[0].root.resolve(strict=True) != root / label: + raise ValueError("the selected diagnostic generation has no exact original root") + parent = selected[0].path(directory) + info = parent.lstat() + row.update(type=_file_type(info.st_mode)) + if not stat.S_ISDIR(info.st_mode): + raise ValueError("the selected diagnostic directory is not an original directory") + known = {database + suffix for selected_directory, database in _fixture_databases(label) if selected_directory == directory for suffix in ("", "-wal", "-shm", "-journal")} + unexpected: list[dict[str, str]] = [] + for path in parent.glob("*.sqlite3*"): + if path.name in known: + continue + if len(unexpected) == 16: + row["truncated"] = True + break + unexpected.append({"basename": path.name, "type": _file_type(path.lstat().st_mode)}) + row.update(status="observed", unexpected=unexpected) + except FileNotFoundError: + row["status"] = "absent" + except (OSError, ValueError) as error: + row.update(status="unavailable", error_type=type(error).__name__) + observations.append(row) + return observations + + def _record_failure_sidecars(result: dict[str, object], chain: Chain | None) -> None: """Keep diagnostic failures from replacing the original native failure.""" if chain is None: return - try: - rows = _original_sidecar_diagnostics(chain.root, tuple(chain.generations)) - result["original_database_sidecars"] = rows - for row in rows: - logger.error("Native original database sidecar metadata: {}", canonical(row).decode()) - except Exception as error: - result["original_database_sidecars"] = [{"status": "unavailable", "error_type": type(error).__name__}] + for key, inspect in (("original_database_sidecars", _original_sidecar_diagnostics), ("original_database_namespaces", _original_namespace_diagnostics)): + try: + rows = inspect(chain.root, tuple(chain.generations)) + result[key] = rows + for row in rows: + logger.error("Native {} metadata: {}", key, canonical(row).decode()) + except Exception as error: + result[key] = [{"status": "unavailable", "error_type": type(error).__name__}] def _checkpoint_stopped(generation: Generation) -> None: @@ -256,7 +302,7 @@ def _finalize_original_stores(chain: Chain, captures: tuple[CapturedGeneration, raise AssertionError("only the three explicitly captured original generations may be finalized") if any(process.poll() != 0 for process in chain.processes): raise AssertionError("an original cutover reader has not completed before finalization") - originals: list[tuple[Path, bytes, os.stat_result]] = [] + originals: list[tuple[Path, bytes, os.stat_result, bool]] = [] for captured in captures: original = captured.original selected = [generation for generation in chain.generations if generation.name == original.generation_id] @@ -269,30 +315,39 @@ def _finalize_original_stores(chain: Chain, captures: tuple[CapturedGeneration, state = json.loads(chain.docker("inspect", "--format", "{{json .State}}", identifier)) if state.get("Status") != "exited" or state.get("Running") is not False or state.get("Pid") != 0 or state.get("Restarting") is not False: raise AssertionError("an original manager or proxy is not concretely stopped before finalization") - for directory, name in ORIGINAL_FIXTURE_DATABASES: + databases = _fixture_databases(generation.name) + for directory in ("state", "journal"): parent = generation.path(directory) - known = {database + suffix for selected_directory, database in ORIGINAL_FIXTURE_DATABASES if selected_directory == directory for suffix in ("", "-wal", "-shm", "-journal")} + known = {database + suffix for selected_directory, database in databases if selected_directory == directory for suffix in ("", "-wal", "-shm", "-journal")} if not stat.S_ISDIR(parent.lstat().st_mode) or {path.name for path in parent.glob("*.sqlite3*")} - known: raise AssertionError("original fixture finalization found an unknown store or directory") - database = parent / name + for directory, name in databases: + database = generation.path(directory) / name encoded, _ = read_original(database) before = database.lstat() + checkpoint = (directory, name) in ORIGINAL_FIXTURE_DATABASES for suffix in ("-wal", "-shm", "-journal"): sidecar = database.with_name(name + suffix) try: info = sidecar.lstat() except FileNotFoundError: continue + if not checkpoint: + raise AssertionError("an original readiness observation still has a transaction sidecar") if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1: raise AssertionError("an original fixture sidecar is not its private regular object") if suffix != "-shm" and info.st_size: raise AssertionError("an original fixture still has nonempty transaction state") - originals.append((database, encoded, before)) + originals.append((database, encoded, before, checkpoint)) # A later archived read may have recreated SQLite's derived SHM index # after an earlier producer checkpoint. Every actual reader is now gone; # the existing checkpoint may close that index, never apply new records. - for database, encoded, before in originals: - Chain.checkpoint(database) + for database, _, _, checkpoint in originals: + if checkpoint: + Chain.checkpoint(database) + # The A/B readiness stores use DELETE mode and need no settlement. Keep + # their observation bytes and original identities just like every store. + for database, encoded, before, _ in originals: current, _ = read_original(database) after = database.lstat() fields = ("st_dev", "st_ino", "st_ctime_ns", "st_mtime_ns", "st_mode", "st_uid", "st_gid", "st_nlink", "st_size") @@ -300,7 +355,7 @@ def _finalize_original_stores(chain: Chain, captures: tuple[CapturedGeneration, raise AssertionError("finalizing a stopped fixture changed original database bytes or identity") if any(database.with_name(database.name + suffix).exists() or database.with_name(database.name + suffix).is_symlink() for suffix in ("-wal", "-shm", "-journal")): raise AssertionError("original fixture sidecars remain after all tracked readers completed") - logger.info("Finalized twelve unchanged original A/B/C stores after all cutover readers stopped, before recovery preparation") + logger.info("Finalized twelve unchanged original A/B/C stores and preserved both A/B readiness observations after all cutover readers stopped, before recovery preparation") def _no_target_child(target: Generation, *, running: bool) -> None: diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index 3b57cf768..df42c2b39 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -129,12 +129,12 @@ def test_failure_sidecar_metadata_identifies_nonempty_shm_without_reading_stores bounded.setattr(Path, "read_bytes", lambda _: pytest.fail("sidecar diagnostics must not read file contents")) bounded.setattr(fixture.sqlite3, "connect", lambda *_args, **_kwargs: pytest.fail("sidecar diagnostics must not connect to SQLite")) rows = fixture._original_sidecar_diagnostics(tmp_path, (generation,)) - assert len(rows) == 12 + assert len(rows) == 15 shm = next(row for row in rows if row["database"] == "management-operations.sqlite3" and row["suffix"] == "-shm") assert shm == {"generation": "A", "database": "management-operations.sqlite3", "suffix": "-shm", "status": "observed", "type": "regular", "size": before.st_size, "device": before.st_dev, "inode": before.st_ino} wal = next(row for row in rows if row["database"] == "management-operations.sqlite3" and row["suffix"] == "-wal") assert wal["status"] == "observed" and wal["size"] == 0 - assert sum(row["status"] == "absent" for row in rows) == 10 + assert sum(row["status"] == "absent" for row in rows) == 13 assert selected.stat() == before assert "private" not in json.dumps(rows) and "oauth" not in json.dumps(rows) and str(tmp_path) not in json.dumps(rows) @@ -148,7 +148,7 @@ def test_failure_sidecar_diagnostics_do_not_follow_links_or_export_other_scopes( (generation.path("journal") / "management-operations.sqlite3-journal").mkdir() unrelated = _diagnostic_generation(tmp_path, "recovery-target") rows = fixture._original_sidecar_diagnostics(tmp_path, (generation, unrelated)) - assert len(rows) == 12 and {row["generation"] for row in rows} == {"A"} + assert len(rows) == 15 and {row["generation"] for row in rows} == {"A"} linked = next(row for row in rows if row.get("type") == "symlink") assert linked["inode"] == sidecar.lstat().st_ino and linked["inode"] != private.stat().st_ino assert any(row.get("type") == "directory" for row in rows) @@ -157,14 +157,49 @@ def test_failure_sidecar_diagnostics_do_not_follow_links_or_export_other_scopes( def test_failure_sidecar_diagnostics_are_finite_and_reject_ambiguous_original_roots(tmp_path: Path) -> None: generations = tuple(_diagnostic_generation(tmp_path, name) for name in ("A", "B", "C", "D")) - assert len(fixture._original_sidecar_diagnostics(tmp_path, generations)) == 48 + assert len(fixture._original_sidecar_diagnostics(tmp_path, generations)) == 54 ambiguous = fixture._original_sidecar_diagnostics(tmp_path, generations + (generations[0],) * 20) - assert len(ambiguous) == 48 + assert len(ambiguous) == 54 assert all(row["status"] == "unavailable" and row["error_type"] == "ValueError" for row in ambiguous if row["generation"] == "A") foreign = Generation("A", tmp_path / "not-original-A", "component-container", []) assert all(row["status"] == "unavailable" for row in fixture._original_sidecar_diagnostics(tmp_path, (foreign,))) +def test_failure_namespace_diagnostics_name_only_bounded_unexpected_sqlite_objects(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + generation = _diagnostic_generation(tmp_path) + state = generation.path("state") + (state / "management-readiness.sqlite3").write_bytes(b"observation must remain private") + (state / "unexpected.sqlite3").write_bytes(b"private unknown data") + (state / "directory.sqlite3").mkdir() + (state / "linked.sqlite3").symlink_to(tmp_path / "private-missing-target") + (state / "unrelated-config.json").write_bytes(b"private configuration") + with monkeypatch.context() as bounded: + bounded.setattr(Path, "read_bytes", lambda _: pytest.fail("namespace diagnostics must not read file contents")) + bounded.setattr(fixture.sqlite3, "connect", lambda *_args, **_kwargs: pytest.fail("namespace diagnostics must not connect to SQLite")) + rows = fixture._original_namespace_diagnostics(tmp_path, (generation,)) + assert len(rows) == 2 + row = next(row for row in rows if row["directory"] == "state") + assert row["status"] == "observed" and row["type"] == "directory" + assert {item["basename"]: item["type"] for item in row["unexpected"]} == {"unexpected.sqlite3": "regular", "directory.sqlite3": "directory", "linked.sqlite3": "symlink"} + assert "private" not in json.dumps(rows) and str(tmp_path) not in json.dumps(rows) + for index in range(40): + (state / f"extra-{index}.sqlite3").touch() + row = next(row for row in fixture._original_namespace_diagnostics(tmp_path, (generation,)) if row["directory"] == "state") + assert row["truncated"] is True and len(row["unexpected"]) == 16 + + +def test_failure_namespace_diagnostics_preserve_c_readiness_absence_and_exact_roots(tmp_path: Path) -> None: + generation = _diagnostic_generation(tmp_path, "C") + (generation.path("state") / "management-readiness.sqlite3").touch() + rows = fixture._original_namespace_diagnostics(tmp_path, (generation,)) + assert rows[0]["unexpected"] == [{"basename": "management-readiness.sqlite3", "type": "regular"}] + assert all(row["status"] == "unavailable" for row in fixture._original_namespace_diagnostics(tmp_path, (generation, generation))) + generation.path("journal").rmdir() + generation.path("journal").symlink_to(generation.path("state")) + row = fixture._original_namespace_diagnostics(tmp_path, (generation,))[1] + assert row["status"] == "unavailable" and row["type"] == "symlink" and "unexpected" not in row + + def test_unavailable_sidecar_diagnostics_preserve_the_original_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: chain = object.__new__(fixture.Chain) chain.root = tmp_path diff --git a/tests/test_management_recovery_fixture_finalization.py b/tests/test_management_recovery_fixture_finalization.py index ee3a70d69..7f4f0925b 100644 --- a/tests/test_management_recovery_fixture_finalization.py +++ b/tests/test_management_recovery_fixture_finalization.py @@ -22,6 +22,7 @@ from auto_coder.management.bootstrap_store import BootstrapStore from auto_coder.management.operations import OperationPhase from auto_coder.management.quarantine_history import QuarantineHistory, QuarantineUnavailable +from auto_coder.management.readiness_observation import _connect as _readiness_store from auto_coder.management.recovery_mailbox import RecoveryMailbox from auto_coder.management.retained_journal import RetainedJournal from auto_coder.management.self_update_handoff import HandoffStore @@ -64,6 +65,12 @@ def originals(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): assert {path.name for path in state.glob("*.sqlite3")} == {"management-child.sqlite3", "management-self-update.sqlite3", "management-recovery.sqlite3"} for path in state.glob("*.sqlite3"): Chain.checkpoint(path) + if name in {"A", "B"}: + # Exercise the production observation-store constructor. This + # component payload makes no authenticated-readiness assertion. + with closing(_readiness_store(SimpleNamespace(state_root=state))) as connection, connection: + assert connection.execute("PRAGMA journal_mode").fetchone() == ("delete",) + connection.execute("INSERT INTO proof VALUES (?, ?)", (name, json.dumps({"component_observation": name}))) generation = Generation(name, root, "manager-" + name, [], proxy="proxy-" + name) chain.generations.append(generation) original = SimpleNamespace(generation_id=name, instance_id=name, state="started", management_id=generation.container, proxy_id=generation.proxy) @@ -94,8 +101,14 @@ def _retained_bytes(chain) -> dict[str, tuple[bytes, int, int, int]]: return result -def test_final_checkpoint_follows_last_archived_read_and_preserves_original_authority(originals) -> None: +def test_final_checkpoint_follows_last_archived_read_and_preserves_original_authority(originals, monkeypatch: pytest.MonkeyPatch) -> None: chain, captures, journals = originals + actual, checkpoints = Chain.checkpoint, [] + + def observed(database: Path) -> None: + checkpoints.append(database) + actual(database) + collection = gc.isenabled() gc.disable() try: @@ -103,10 +116,14 @@ def test_final_checkpoint_follows_last_archived_read_and_preserves_original_auth before = _retained_bytes(chain) with pytest.raises(QuarantineUnavailable, match="nonempty transaction sidecar"): QuarantineHistory((journals["A"],)) + monkeypatch.setattr(Chain, "checkpoint", observed) fixture._finalize_original_stores(chain, captures) after = _retained_bytes(chain) assert {name: value for name, value in before.items() if not name.endswith(("-wal", "-shm", "-journal"))} == after assert (chain.root / "D" / "retained-never-created").read_bytes() == b"original failed branch" + assert len(checkpoints) == 12 and {path.name for path in checkpoints} == {"management-child.sqlite3", "management-self-update.sqlite3", "management-recovery.sqlite3", "management-operations.sqlite3"} + assert len([name for name in after if name.endswith(".sqlite3")]) == 14 + assert not (chain.root / "C" / "state" / "management-readiness.sqlite3").exists() for name, journal in journals.items(): with QuarantineHistory((journal,)) as history: report = history.inspect() @@ -130,6 +147,41 @@ def test_all_real_original_mailboxes_are_required_and_never_reconstructed(origin assert not missing.exists() and _retained_bytes(chain) == before +@pytest.mark.parametrize("generation", ["A", "B"]) +def test_completed_readiness_observations_are_required_and_never_reconstructed(originals, generation: str) -> None: + chain, captures, journals = originals + _late_archived_read(journals["A"]) + missing = chain.root / generation / "state" / "management-readiness.sqlite3" + missing.unlink() + before = _retained_bytes(chain) + with pytest.raises(FileNotFoundError): + fixture._finalize_original_stores(chain, captures) + assert not missing.exists() and _retained_bytes(chain) == before + + +def test_unresolved_cannot_acquire_a_readiness_observation_during_finalization(originals) -> None: + chain, captures, journals = originals + _late_archived_read(journals["A"]) + with closing(_readiness_store(SimpleNamespace(state_root=chain.root / "C" / "state"))): + pass + before = _retained_bytes(chain) + with pytest.raises(AssertionError, match="unknown store"): + fixture._finalize_original_stores(chain, captures) + assert _retained_bytes(chain) == before + + +@pytest.mark.parametrize("suffix", ["-wal", "-shm", "-journal"]) +def test_observation_sidecars_refuse_without_settling_any_original_store(originals, suffix: str) -> None: + chain, captures, journals = originals + _late_archived_read(journals["A"]) + sidecar = chain.root / "B" / "state" / ("management-readiness.sqlite3" + suffix) + sidecar.touch(mode=0o600) + before = _retained_bytes(chain) + with pytest.raises(AssertionError, match="readiness observation still has a transaction sidecar"): + fixture._finalize_original_stores(chain, captures) + assert _retained_bytes(chain) == before + + @pytest.mark.parametrize("fault", ["nonempty-wal", "hot-journal", "missing-store", "unknown-store", "linked-sidecar", "running-manager", "running-proxy", "live-cutover", "published-inventory", "installed-authorization", "target-prepared", "wrong-capture", "missing-capture"]) def test_finalization_refuses_uncertain_scope_before_any_checkpoint(originals, fault: str) -> None: chain, captures, journals = originals From fd141fff36ec05776e9995efacd4e9c7a82fd445 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 03:19:11 +0900 Subject: [PATCH 14/24] test: retain real scheduling in concurrent owner cancellation --- tests/test_invocation_process_supervisor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_invocation_process_supervisor.py b/tests/test_invocation_process_supervisor.py index c9b0d8973..5de2c052e 100644 --- a/tests/test_invocation_process_supervisor.py +++ b/tests/test_invocation_process_supervisor.py @@ -365,7 +365,8 @@ def test_timeout_and_confirmation_uncertainty_remain_distinct(tmp_path: Path) -> assert supervisor.settle_retained(result.invocation_id) -def test_cancelling_one_owner_does_not_stop_peer_or_delete_results(tmp_path: Path) -> None: +def test_cancelling_one_owner_does_not_stop_peer_or_delete_results(tmp_path: Path, _use_real_sleep) -> None: + # Actual concurrent owners and readiness polling must yield between observations. supervisor = make_supervisor(ProcessGroupOwner(tmp_path / "owners")) cancel = threading.Event() first_result = tmp_path / "first-result" From 4dbab2489020e4db40c127d3ed3cb5f6bd4be5b4 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 03:35:46 +0900 Subject: [PATCH 15/24] fix(management): retain bounded refusal cause diagnostics --- .../management-recovery-epoch.md | 5 + src/auto_coder/management/recovery_epoch.py | 27 +++++ ...t_management_recovery_epoch_diagnostics.py | 99 +++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 tests/test_management_recovery_epoch_diagnostics.py diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md index aee7b54f3..fe9898012 100644 --- a/docs/client-features/management-recovery-epoch.md +++ b/docs/client-features/management-recovery-epoch.md @@ -186,6 +186,11 @@ GitHub issue/PR processing attempt, dashboard slot, management operation admissi or worker outcome is created by this host ownership protocol. See the matching entry in `docs/dashboard-observability.md`. +Refusal diagnostics retain up to eight explicit exception causes with only their +type, integer error number, and code basename/function/line. They omit inner +exception messages, arguments, frame locals and full paths, and never change the +refusal decision or preserved state. + The maintained component selections cover real persistence failures/process interruption, exact fixed-scope parsing, independent-process SH/EX/nsfs custody, actual received descriptors, current peer lifetime and cancellation. Component diff --git a/src/auto_coder/management/recovery_epoch.py b/src/auto_coder/management/recovery_epoch.py index 5a3543edd..ef1d7090d 100644 --- a/src/auto_coder/management/recovery_epoch.py +++ b/src/auto_coder/management/recovery_epoch.py @@ -11,6 +11,7 @@ import hashlib import json import os +import re import signal import socket import stat @@ -360,6 +361,30 @@ def run_issuer() -> EpochDispatch: signal.signal(saved_signal, handler) # type: ignore[arg-type] +def _refusal_causes(error: BaseException) -> list[dict[str, str | int]]: + """Describe bounded code locations, never exception data or frame locals.""" + causes: list[dict[str, str | int]] = [] + visited: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in visited and len(causes) < 8: + visited.add(id(current)) + name = type(current).__name__ + row: dict[str, str | int] = {"type": name if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,95}", name) else "Exception"} + if isinstance(current, OSError) and type(current.errno) is int: + row["errno"] = current.errno + trace = current.__traceback__ + if trace is not None: + while trace.tb_next is not None: + trace = trace.tb_next + code = trace.tb_frame.f_code + filename = Path(code.co_filename).name + if re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}\.py", filename) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,95}|", code.co_name): + row.update(file=filename, function=code.co_name, line=trace.tb_lineno) + causes.append(row) + current = current.__cause__ + return causes + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Issue one approved management ownership epoch from fixed host authority") parser.parse_args(argv) @@ -368,6 +393,8 @@ def main(argv: list[str] | None = None) -> int: result = run_issuer() except (OSError, ValueError, TypeError, KeyError, RuntimeError) as error: logger.bind(recovery_epoch_phase="unavailable", current_ready=False).error("Recovery epoch unavailable; preserved evidence is not repaired: {}", error) + for index, cause in enumerate(_refusal_causes(error), start=1): + logger.bind(recovery_epoch_phase="unavailable", current_ready=False).error("Recovery epoch refusal cause {}: {}", index, json.dumps(cause, sort_keys=True, separators=(",", ":"))) return 2 print(json.dumps(asdict(result), sort_keys=True)) return 0 diff --git a/tests/test_management_recovery_epoch_diagnostics.py b/tests/test_management_recovery_epoch_diagnostics.py new file mode 100644 index 000000000..6b96e5477 --- /dev/null +++ b/tests/test_management_recovery_epoch_diagnostics.py @@ -0,0 +1,99 @@ +"""Bounded issuer-refusal diagnostics; no native acceptance authority.""" + +from __future__ import annotations + +import errno +import json +import re +from pathlib import Path + +import pytest + +from auto_coder.logger_config import setup_logger +from auto_coder.management import recovery_epoch as issuer + + +def _failure() -> ValueError: + # The code location is safe metadata; its directory, exception arguments, + # attached filename and frame locals are deliberately private test data. + source = "def fail():\n local_credential = 'private-local-credential'\n raise FileNotFoundError(2, 'private-error-message', '/private/source-token.txt')\n" + namespace = {} + exec(compile(source, "/private/code-directory/observed_component.py", "exec"), namespace) + try: + try: + namespace["fail"]() + except OSError as cause: + raise ValueError("fixed issuer refusal") from cause + except ValueError as error: + return error + + +def test_refusal_causes_retain_only_code_identity_and_integer_errno() -> None: + error = _failure() + rows = issuer._refusal_causes(error) + assert rows[0] == {"type": "ValueError", "file": Path(__file__).name, "function": "_failure", "line": error.__traceback__.tb_lineno} + assert rows[1] == {"type": "FileNotFoundError", "errno": errno.ENOENT, "file": "observed_component.py", "function": "fail", "line": 3} + assert "private" not in json.dumps(rows) and "fixed issuer refusal" not in json.dumps(rows) + assert all(set(row) <= {"type", "errno", "file", "function", "line"} for row in rows) + + +def test_refusal_cause_chain_is_limited_to_eight_entries() -> None: + error = ValueError("private-deep-message") + for _ in range(40): + outer = RuntimeError("private-outer-message") + outer.__cause__ = error + error = outer + assert issuer._refusal_causes(error) == [{"type": "RuntimeError"}] * 8 + + +@pytest.mark.parametrize("self_cycle", [False, True]) +def test_refusal_cause_cycles_are_finite(self_cycle: bool) -> None: + first, second = ValueError("private-first"), OSError(errno.EIO, "private-second") + first.__cause__ = first if self_cycle else second + second.__cause__ = first + assert issuer._refusal_causes(first) == ([{"type": "ValueError"}] if self_cycle else [{"type": "ValueError"}, {"type": "OSError", "errno": errno.EIO}]) + + +@pytest.mark.parametrize("number", ["private-errno", True, None]) +def test_refusal_causes_ignore_noninteger_errno(number: object) -> None: + error = OSError("private-argument") + error.errno = number + assert issuer._refusal_causes(error) == [{"type": "OSError"}] + + +def test_refusal_causes_never_format_exception_objects_or_unsafe_code_metadata() -> None: + class SensitiveFailure(ValueError): + def __str__(self) -> str: + pytest.fail("causal metadata must not format the exception") + + try: + exec(compile("raise error", "/private/path/invalid filename.py", "exec"), {"error": SensitiveFailure("private-argument")}) + except SensitiveFailure as error: + assert issuer._refusal_causes(error) == [{"type": "SensitiveFailure"}] + + +def test_cli_refusal_preserves_exit_and_emits_safe_causes_to_stdout_and_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + log = tmp_path / "issuer.log" + + def configure(**kwargs: object) -> None: + kwargs["log_file"] = str(log) + setup_logger(**kwargs) + + def refuse() -> None: + raise _failure() + + monkeypatch.setattr(issuer, "setup_logger", configure) + monkeypatch.setattr(issuer, "run_issuer", refuse) + assert issuer.main([]) == 2 + issuer.logger.complete() + standard = re.sub(r"\x1b\[[0-9;]*m", "", capsys.readouterr().out) + persisted = log.read_text() + for output in (standard, persisted): + assert "Recovery epoch unavailable; preserved evidence is not repaired: fixed issuer refusal" in output + messages = [line.partition("Recovery epoch refusal cause ")[2] for line in output.splitlines() if "Recovery epoch refusal cause " in line] + assert len(messages) == 2 + assert [line.partition(": ")[0] for line in messages] == ["1", "2"] + rows = [json.loads(line.partition(": ")[2]) for line in messages] + assert rows[1] == {"type": "FileNotFoundError", "errno": errno.ENOENT, "file": "observed_component.py", "function": "fail", "line": 3} + assert "private" not in output + issuer.logger.remove() From b97d1683208cbee85c34ebc5b262fa11d820b2e7 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 04:01:57 +0900 Subject: [PATCH 16/24] fix(test): stabilize native recovery Docker route catalog --- scripts/recovery_vm/README.md | 21 +++++ scripts/recovery_vm/guest.sh | 81 +++++++++++++++++++ .../test_management_recovery_epoch_host.py | 32 +++++++- tests/test_management_recovery_epoch_host.py | 36 +++++++++ 4 files changed, 169 insertions(+), 1 deletion(-) diff --git a/scripts/recovery_vm/README.md b/scripts/recovery_vm/README.md index a7b06b5cc..aaaee3675 100644 --- a/scripts/recovery_vm/README.md +++ b/scripts/recovery_vm/README.md @@ -30,6 +30,27 @@ another image or weaken the production source checks. Docker documents the [fresh-install storage default](https://docs.docker.com/engine/storage/containerd/) and the [cross-store image-ID behavior](https://github.com/docker/cli/issues/6670). +The selected guest service profile is ordinary system Docker with its standard +daemon-managed, separate-process containerd and `PrivateMounts=yes`. Before package +installation can start an engine, preparation masks Docker and the standalone +containerd service. It then installs the reviewed Docker override, unmasks Docker +only, and checks the actual daemon command, PID/start identity, systemd cgroup +driver, empty engine and managed containerd child. Both daemons must share one +mount namespace distinct from PID 1 while retaining the host PID, user and network +namespaces. These observations are retained in `docker-profile.json`, and the +effective service configuration is included in `guest-platform.txt`. + +This mount profile keeps transient container mounts out of the host systemd unit +catalog throughout target starts and restarts; production still requires exact +equality with the approved catalog and refuses unknown routes. Docker documents +[automatic containerd startup](https://docs.docker.com/reference/cli/dockerd/#run-containerd-standalone), +and systemd 255 documents +[PrivateMounts propagation](https://www.freedesktop.org/software/systemd/man/255/systemd.exec.html#PrivateMounts=). +This is provisioning of the fresh synthetic VM before any original history or +recovery approval exists. It does not change an operational host, configure the +production issuer, or establish acceptance for every stock Docker service profile +or an incident with missing originals. + The guest installs the exact `uv==0.13.6` release and validates its reported release number. The complete observed `--version` banner, including its bounded build descriptor, is retained in `uv-version.txt` and supplied to staging and recovery diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh index 33d275403..82ebb450a 100644 --- a/scripts/recovery_vm/guest.sh +++ b/scripts/recovery_vm/guest.sh @@ -14,6 +14,8 @@ finish() { { systemctl cat cloud-final.service cloud-config.service cloud-init.service cloud-init-local.service systemctl show cloud-final.service cloud-config.service cloud-init.service cloud-init-local.service -p ExecStart -p ActiveState -p UnitFileState + systemctl cat docker.service docker.socket containerd.service + systemctl show docker.service containerd.service -p ExecStart -p PrivateMounts -p MainPID -p ActiveState -p LoadState -p UnitFileState -p FragmentPath -p DropInPaths docker version docker compose version } > "$output/guest-platform.txt" 2>&1 @@ -45,9 +47,88 @@ mkdir -p /etc/docker test ! -e /etc/docker/daemon.json test ! -L /etc/docker/daemon.json install -m 600 "$input/docker-daemon.json" /etc/docker/daemon.json +# This fresh VM has no previous engine. Prevent package auto-start before the +# selected ordinary service profile exists; no original history exists yet. +test ! -e /usr/bin/dockerd +test ! -e /run/docker.pid +test ! -e /var/lib/docker +for fixture_unit in docker.service docker.socket containerd.service; do + test ! -e "/etc/systemd/system/$fixture_unit" + test ! -L "/etc/systemd/system/$fixture_unit" + ln -s /dev/null "/etc/systemd/system/$fixture_unit" +done +systemctl daemon-reload apt-get update apt-get install -y --no-install-recommends ca-certificates cron curl docker.io docker-compose-v2 git python3-venv +mkdir -p /etc/systemd/system/docker.service.d +test ! -e /etc/systemd/system/docker.service.d/recovery-fixture.conf +tee /etc/systemd/system/docker.service.d/recovery-fixture.conf > /dev/null <<'UNIT' +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd -H fd:// +PrivateMounts=yes +UNIT +# Docker starts its normal managed containerd process in this same namespace. +# A standalone containerd in a different mount namespace could not see its +# overlay/netns objects. Keep that unused packaged service masked throughout. +for fixture_unit in docker.service docker.socket; do + test "$(readlink "/etc/systemd/system/$fixture_unit")" = /dev/null + rm "/etc/systemd/system/$fixture_unit" +done +systemctl daemon-reload systemctl start docker +python3 - "$output/docker-profile.json" <<'PY' +import json +import os +from pathlib import Path +import subprocess +import sys + + +def properties(unit): + names = ("MainPID", "ActiveState", "LoadState", "PrivateMounts") + raw = subprocess.check_output(["systemctl", "show", unit, "--property=" + ",".join(names)], text=True) + result = dict(line.split("=", 1) for line in raw.splitlines()) + assert set(result) == set(names) + return result + + +def process(pid): + proc = Path("/proc") / str(pid) + fields = (proc / "stat").read_text().rsplit(")", 1)[1].split() + assert fields[0] not in ("Z", "X", "x") and int(fields[19]) > 0 + return { + "pid": pid, + "parent": int(fields[1]), + "start": fields[19], + "namespaces": {name: os.readlink(proc / "ns" / name) for name in ("pid", "user", "net", "mnt")}, + } + + +service, standalone = properties("docker.service"), properties("containerd.service") +assert service["ActiveState"] == "active" and service["PrivateMounts"] == "yes" +assert standalone["LoadState"] == "masked" and standalone["ActiveState"] == "inactive" and standalone["MainPID"] == "0" +pid = int(service["MainPID"]) +assert pid > 1 and Path("/run/docker.pid").read_text().strip() == str(pid) +assert os.path.samefile(f"/proc/{pid}/exe", "/usr/bin/dockerd") +assert Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0") == [b"/usr/bin/dockerd", b"-H", b"fd://", b""] +daemon = process(pid) +host = process(1) +assert all(daemon["namespaces"][name] == host["namespaces"][name] for name in ("pid", "user", "net")) +assert daemon["namespaces"]["mnt"] != host["namespaces"]["mnt"] +children = {int(value) for task in Path(f"/proc/{pid}/task").iterdir() for value in (task / "children").read_text().split()} +managed = [child for child in children if os.path.samefile(f"/proc/{child}/exe", "/usr/bin/containerd")] +assert len(managed) == 1 +containerd = process(managed[0]) +assert containerd["parent"] == pid and containerd["namespaces"] == daemon["namespaces"] +assert subprocess.check_output(["docker", "info", "--format", "{{.CgroupDriver}}"], text=True).strip() == "systemd" +assert subprocess.check_output(["docker", "ps", "--all", "--quiet"], text=True).strip() == "" +assert process(pid) == daemon and process(managed[0]) == containerd +result = {"schema": "auto-coder-recovery-vm-docker-profile-v1", "profile": "ordinary-system-docker-managed-containerd-private-mounts", "daemon": daemon, "managed_containerd": containerd, "standalone_service": standalone} +with Path(sys.argv[1]).open("x") as stream: + json.dump(result, stream, sort_keys=True) +print("Verified ordinary Docker with its managed containerd in one private mount namespace before original fixture creation.", flush=True) +PY docker info --format 'Actual guest image store: {{.Driver}} {{json .DriverStatus}}' python3 -m venv /opt/fixture-tools /opt/fixture-tools/bin/pip install uv==0.13.6 diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 16049360e..efa4a5f57 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -39,12 +39,13 @@ from auto_coder.logger_config import get_logger, setup_logger from auto_coder.management.bootstrap_supervisor import BootstrapConfig from auto_coder.management.bootstrap_upgrade import _source_authority +from auto_coder.management.containment_route_manifest import SystemdRoute, _systemd_catalog from auto_coder.management.fresh_initialization import FreshInitializationIntent, claim_fresh_initialization, read_fresh_status from auto_coder.management.operation_store import ManagementOperationStore from auto_coder.management.operations import ActionParameter, ManagedTarget, OperationIntent, OperationPhase from auto_coder.management.original_attempt_evidence import derive_lineage, observe, verify_lineage from auto_coder.management.original_attempts import OriginalEvidenceUnavailable, canonical, read_original, source_identity -from auto_coder.management.recovery_containment_model import RecoveryWatcher +from auto_coder.management.recovery_containment_model import RECOVERY_ENROLLMENT_ROOT, RecoveryWatcher from auto_coder.management.recovery_epoch_authorization import AUTHORIZATION_FILE, AUTHORIZATION_ROOT, AUTHORIZATION_SCHEMA, PEER_ROOT, PEER_SOCKET, EpochAuthorization, validate_uv_version from auto_coder.management.recovery_epoch_consumer import GRANT_SCHEMA, ConsumerConfiguration, OriginalFence, TargetOwner, TransferredResource from auto_coder.management.recovery_epoch_state import EPOCH_ROOT, EpochInventory @@ -287,6 +288,34 @@ def _record_failure_sidecars(result: dict[str, object], chain: Chain | None) -> result[key] = [{"status": "unavailable", "error_type": type(error).__name__}] +def _record_failure_catalog(result: dict[str, object], enrollment_id: str | None) -> None: + """Record a bounded post-failure catalog delta without changing approval.""" + if enrollment_id is None: + return + try: + if not re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", enrollment_id): + raise ValueError("invalid diagnostic enrollment selector") + encoded, _ = read_original(RECOVERY_ENROLLMENT_ROOT / (enrollment_id + ".routes.json")) + if len(encoded) > 1024 * 1024: + raise ValueError("oversized diagnostic route manifest") + manifest = json.loads(encoded) + routes = manifest.get("systemd_routes") + if manifest.get("enrollment_id") != enrollment_id or not isinstance(routes, list) or not 1 <= len(routes) <= 1024: + raise ValueError("diagnostic route selection differs") + expected = {SystemdRoute.parse(row).unit for row in routes} + if len(expected) != len(routes): + raise ValueError("duplicate diagnostic route selection") + current = set(_systemd_catalog()) + if any(len(unit) > 256 for unit in expected | current): + raise ValueError("oversized diagnostic unit name") + added, removed = sorted(current - expected), sorted(expected - current) + observed = {"status": "observed-after-failure", "approved_count": len(expected), "observed_count": len(current), "added_count": len(added), "removed_count": len(removed), "added": added[:32], "removed": removed[:32], "truncated": len(added) > 32 or len(removed) > 32} + result["systemd_catalog_after_failure"] = observed + logger.error("Native systemd catalog after failure: {}", canonical(observed).decode()) + except Exception as error: + result["systemd_catalog_after_failure"] = {"status": "unavailable", "error_type": type(error).__name__} + + def _checkpoint_stopped(generation: Generation) -> None: """The stopped fixture producer settles its own stores before inspection.""" for directory in (generation.path("state"), generation.path("journal")): @@ -1301,6 +1330,7 @@ def preservation() -> object: result["status"] = "failed" result["failure_type"] = type(error).__name__ _record_failure_sidecars(result, chain) + _record_failure_catalog(result, target.enrollment_id if target is not None else None) command_failure = _command_failure(error) if command_failure is not None: result["command_failure"] = command_failure diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index df42c2b39..116512716 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -222,6 +222,42 @@ def unavailable(*_args): assert "private" not in json.dumps(result) +def test_failure_catalog_delta_names_exact_units_without_exporting_manifest_contents(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fixture, "RECOVERY_ENROLLMENT_ROOT", tmp_path) + path = tmp_path / "component.routes.json" + path.write_bytes(canonical({"enrollment_id": "component", "systemd_routes": [{"unit": name, "sha256": "a" * 64, "role": "unrelated"} for name in ("kept.service", "removed.mount")], "private_other_data": "private-source-credential"})) + path.chmod(0o600) + before = path.read_bytes(), path.stat() + monkeypatch.setattr(fixture, "_systemd_catalog", lambda: ("added.mount", "kept.service")) + result = {"status": "failed", "failure_type": "AssertionError"} + fixture._record_failure_catalog(result, "component") + assert result["systemd_catalog_after_failure"] == {"status": "observed-after-failure", "approved_count": 2, "observed_count": 2, "added_count": 1, "removed_count": 1, "added": ["added.mount"], "removed": ["removed.mount"], "truncated": False} + assert "private" not in json.dumps(result) and str(tmp_path) not in json.dumps(result) + after = path.stat() + assert path.read_bytes() == before[0] and all(getattr(after, field) == getattr(before[1], field) for field in ("st_dev", "st_ino", "st_ctime_ns", "st_mtime_ns", "st_size", "st_mode")) and result["status"] == "failed" + monkeypatch.setattr(fixture, "_systemd_catalog", lambda: tuple(f"new-{index:02d}.mount" for index in range(50))) + fixture._record_failure_catalog(result, "component") + observed = result["systemd_catalog_after_failure"] + assert observed["added_count"] == 50 and len(observed["added"]) == 32 and observed["truncated"] is True + + +def test_failure_catalog_diagnostic_errors_never_replace_original_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(fixture, "RECOVERY_ENROLLMENT_ROOT", tmp_path) + result = {"status": "failed", "failure_type": "AssertionError"} + original = AssertionError("original native refusal") + with pytest.raises(AssertionError) as caught: + try: + raise original + except AssertionError: + fixture._record_failure_catalog(result, "../private") + raise + assert caught.value is original + assert result["systemd_catalog_after_failure"] == {"status": "unavailable", "error_type": "ValueError"} + assert "private" not in json.dumps(result) + fixture._record_failure_catalog(result, "missing") + assert result["systemd_catalog_after_failure"] == {"status": "unavailable", "error_type": "FileNotFoundError"} + + def test_no_child_or_admission_oracle_rejects_any_native_store(tmp_path: Path) -> None: state, journal = tmp_path / "state", tmp_path / "journal" state.mkdir() From 65b1cc216f3f9046ded04c82778409ebf73a0401 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 04:03:13 +0900 Subject: [PATCH 17/24] fix(test): observe fresh readiness after authenticated readback --- ...t_management_archived_cutover_container.py | 18 +++- ...t_management_archived_cutover_readiness.py | 91 +++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 tests/test_management_archived_cutover_readiness.py diff --git a/tests/test_management_archived_cutover_container.py b/tests/test_management_archived_cutover_container.py index ea569f16e..b5899cdd4 100644 --- a/tests/test_management_archived_cutover_container.py +++ b/tests/test_management_archived_cutover_container.py @@ -533,6 +533,10 @@ def call(self, generation: Generation, tool: str, arguments: dict[str, object], return {"http_status": error.code} raise ValueError("synthetic authenticated proxy HTTP unavailable") from None + def fresh_status(self, generation: Generation) -> dict[str, object]: + code = "import json;from pathlib import Path;from auto_coder.management.fresh_initialization import read_fresh_status;print(json.dumps(read_fresh_status(Path('/management-child-state'))))" + return json.loads(self.docker("exec", generation.container, "python", "-c", code, timeout=10).splitlines()[-1]) + def ready(self, generation: Generation, *, fresh: bool) -> None: self.ensure_proxy(generation) deadline = time.monotonic() + 60 @@ -541,8 +545,7 @@ def ready(self, generation: Generation, *, fresh: bool) -> None: try: observation = "" if fresh: - code = "import json;from pathlib import Path;from auto_coder.management.fresh_initialization import read_fresh_status;print(json.dumps(read_fresh_status(Path('/management-child-state'))))" - status = json.loads(self.docker("exec", generation.container, "python", "-c", code, timeout=10).splitlines()[-1]) + status = self.fresh_status(generation) last = "fresh initialization status=" + str(status["status"]) if status["status"] not in {"launched", "succeeded"}: time.sleep(0.1) @@ -554,7 +557,16 @@ def ready(self, generation: Generation, *, fresh: bool) -> None: continue last = "authenticated Caddy whoami" result = self.call(generation, "auto_coder_whoami", {"observation_id": observation}) - if result["uid"] == "operator" and (not fresh or status["status"] == "succeeded"): + confirmed = result["uid"] == "operator" + if confirmed and fresh: + # Authenticated I/O can complete readiness after the initial + # sample. Require a current terminal result for that same launch. + terminal = self.fresh_status(generation) + last = "fresh initialization terminal status=" + str(terminal["status"]) + if {key: value for key, value in terminal.items() if key != "status"} != {key: value for key, value in status.items() if key != "status"}: + raise ValueError("fresh initialization launch changed during authenticated readback") + confirmed = terminal["status"] == "succeeded" + if confirmed: if self.worker_baseline is not None: last = "authenticated worker identity readback" self.assert_worker_reads(generation) diff --git a/tests/test_management_archived_cutover_readiness.py b/tests/test_management_archived_cutover_readiness.py new file mode 100644 index 000000000..5bcacd8dd --- /dev/null +++ b/tests/test_management_archived_cutover_readiness.py @@ -0,0 +1,91 @@ +"""Component checks for the native fixture's readiness observation order. + +Controlled replies exercise the fixture oracle only; they are not evidence of +Docker, authenticated application readiness, or a successful native cutover. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests import test_management_archived_cutover_container as cutover + + +def _ready_case(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, initial_status: str = "launched", terminal_status: str = "succeeded", operator: bool = True, changed_field: str | None = None, challenge: int = 401): + generation = cutover.Generation("manager-c", tmp_path, "exact-manager-c", [], proxy="exact-proxy-c") + chain = cutover.Chain.__new__(cutover.Chain) + chain.worker_baseline = None + launch = {"request_id": "request-c", "owner_incarnation": "owner-c", "child_identity": "child-c", "observation_id": "a" * 32, "started": 100.0, "deadline": 160.0} + current = [{**launch, "status": initial_status}] + reads: list[dict[str, object]] = [] + authenticated_calls: list[bool] = [] + + def docker(*arguments, **kwargs): + if arguments[0] == "exec": + assert arguments[1] == generation.container + reads.append(dict(current[0])) + return json.dumps(current[0]) + assert arguments[0] == "inspect" + return "running" + + def call(selected, tool, arguments, *, authenticated=True): + assert selected is generation and tool == "auto_coder_whoami" + authenticated_calls.append(authenticated) + if not authenticated: + return {"http_status": challenge} + assert arguments == {"observation_id": launch["observation_id"]} + current[0] = {**launch, "status": terminal_status} + if changed_field is not None: + current[0][changed_field] = "different-generation" + return {"uid": "operator" if operator else "different-principal"} + + # A single poll starts within the existing budget. A second poll would be + # too late, reproducing the completed transition during authenticated I/O. + ticks = iter((0.0, 1.0, 61.0)) + monkeypatch.setattr(cutover, "time", SimpleNamespace(monotonic=lambda: next(ticks, 61.0), sleep=lambda _: None)) + monkeypatch.setattr(chain, "ensure_proxy", lambda selected: None) + monkeypatch.setattr(chain, "docker", docker) + monkeypatch.setattr(chain, "call", call) + return chain, generation, reads, authenticated_calls + + +@pytest.mark.parametrize("initial_status", ["launched", "succeeded"]) +def test_terminal_read_follows_authenticated_whoami(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, initial_status: str) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, initial_status=initial_status) + chain.ready(generation, fresh=True) + assert [value["status"] for value in reads] == [initial_status, "succeeded"] + assert calls == [False, True] + + +@pytest.mark.parametrize("changed_field", ["observation_id", "child_identity"]) +def test_terminal_read_must_bind_the_same_authenticated_launch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, changed_field: str) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, changed_field=changed_field) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.ready(generation, fresh=True) + assert len(reads) == 2 and calls == [False, True] + + +def test_authenticated_whoami_does_not_replace_terminal_success(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, terminal_status="launched") + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.ready(generation, fresh=True) + assert [value["status"] for value in reads] == ["launched", "launched"] + assert calls == [False, True] + + +def test_terminal_success_does_not_replace_operator_identity(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, operator=False) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.ready(generation, fresh=True) + assert len(reads) == 1 and calls == [False, True] + + +def test_unauthenticated_challenge_must_still_refuse_access(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, reads, calls = _ready_case(monkeypatch, tmp_path, challenge=200) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.ready(generation, fresh=True) + assert len(reads) == 1 and calls == [False] From db3e3d7aa77c2058194170e7da15da744bc8916e Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 04:17:41 +0900 Subject: [PATCH 18/24] fix(management): separate recovery approval from unrelated cgroups --- .../management-recovery-epoch.md | 9 ++ .../management/containment_route_manifest.py | 29 +++- ..._management_containment_cron_inhibition.py | 140 ++++++++++++++++++ 3 files changed, 175 insertions(+), 3 deletions(-) diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md index fe9898012..ad11714fa 100644 --- a/docs/client-features/management-recovery-epoch.md +++ b/docs/client-features/management-recovery-epoch.md @@ -50,6 +50,15 @@ hold under separate reviewed operational authorization. Unknown controller route or unsupported profiles refuse. NEW/#2672 and the existing ordinary/quarantined observation profiles retain their admission contracts. +Recovery route approval binds the exact cron sources, masks, controller +configuration, boot, cgroup mount, and every inhibited scope's required roots, +absence, empty-task facts and directory identities, including their ancestors. +An unrelated cgroup directory, such as the selected target's ordinary Docker +scope, may appear or be recreated after approval. Every current observation still +enumerates the complete cgroup tree and all tasks, retains those unrelated +directory identities, and requires the full repeated observations to match. +The ordinary route profiles retain their exact persisted-snapshot comparison. + ## Deployment-fixed authority The host CLI is `python -m auto_coder.management.recovery_epoch`. It takes no diff --git a/src/auto_coder/management/containment_route_manifest.py b/src/auto_coder/management/containment_route_manifest.py index 445096608..1f9686f4a 100644 --- a/src/auto_coder/management/containment_route_manifest.py +++ b/src/auto_coder/management/containment_route_manifest.py @@ -17,7 +17,7 @@ import re import stat import subprocess -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from pathlib import Path from .archived_exclusions import unique_pairs @@ -893,6 +893,28 @@ def _validate_user_scope(manifest: ApprovedRouteManifest, inventory: UserControl raise RouteManifestUnavailable("installed user management source lacks an independently inhibited unit") +def _cron_approval_binding(entries: tuple[CronEntry, ...], *, recovery: bool) -> tuple[CronEntry, ...]: + """Bind recovery approval to every inhibited scope, not unrelated groups. + + A target's ordinary Docker scope can be created after route approval. Only + unrelated cgroup directory identities are excluded from this historical + comparison. Complete fresh snapshots and returned observations retain them. + Ordinary profiles keep their existing exact approval comparison. + """ + if not recovery: + return entries + bindings = [] + for entry in entries: + source = entry.inhibited_source + if source is None: + bindings.append(entry) + continue + scope = source.cgroups + directories = tuple(item for item in scope.directories if item.path in ("/", "/system.slice") or any(item.path == root or item.path.startswith(root + "/") or root.startswith(item.path.rstrip("/") + "/") for root in scope.required_empty_roots)) + bindings.append(replace(entry, inhibited_source=replace(source, cgroups=replace(scope, directories=directories)))) + return tuple(bindings) + + def _observe_approved_route_manifest( approval: EnrollmentObservation | RecoveryRouteSelection, docker: tuple[object, ...], @@ -921,7 +943,8 @@ def _observe_approved_route_manifest( if ids != manifest.docker_container_ids: raise RouteManifestUnavailable("whole-engine Docker inventory differs from approved routes") is_closed_version = manifest.schema in ("approved-management-ordinary-routes-v2", "approved-management-recovery-routes-v1") - if isinstance(approval, RecoveryRouteSelection) != (manifest.schema == "approved-management-recovery-routes-v1"): + recovery_routes = manifest.schema == "approved-management-recovery-routes-v1" + if isinstance(approval, RecoveryRouteSelection) != recovery_routes: raise RouteManifestUnavailable("route manifest and selection profile differ") bindings = _deployment_root_bindings(approval, trusted_uid=expected_uid) if is_closed_version else () if bindings != manifest.deployment_roots: @@ -943,7 +966,7 @@ def _observe_approved_route_manifest( if manifest.cron_restarter_paths and not _CRON_SCHEDULERS.intersection(restarters): raise RouteManifestUnavailable("ordinary cron restarter lacks scheduler inhibition") cron = _cron_catalog(roots=cron_roots, trusted_uid=expected_uid, extra_cues=cues, inspect_local_inputs=is_closed_version, module_sources=modules, source_formats=cron_source_formats, systemd_facts=observed_systemd) - if cron != manifest.cron_entries: + if _cron_approval_binding(cron, recovery=recovery_routes) != _cron_approval_binding(manifest.cron_entries, recovery=recovery_routes): raise RouteManifestUnavailable("unknown, omitted or changed cron/at route") if any(entry.management_candidate and entry.path not in manifest.cron_restarter_paths for entry in cron): raise RouteManifestUnavailable("detected management cron/at route is excluded") diff --git a/tests/test_management_containment_cron_inhibition.py b/tests/test_management_containment_cron_inhibition.py index bf6f51939..a66511f35 100644 --- a/tests/test_management_containment_cron_inhibition.py +++ b/tests/test_management_containment_cron_inhibition.py @@ -12,6 +12,7 @@ import pwd import stat import tempfile +from contextlib import contextmanager from dataclasses import asdict, replace from pathlib import Path from types import SimpleNamespace @@ -54,6 +55,35 @@ def _scope(tree, units: tuple[str, ...] = ("cron.service",)) -> cron.CronCgroupS return cron._cgroup_snapshot(root=root, proc=proc, mountinfo=mountinfo, inhibited_units=units) +def _stock_entry(proof: cron.CronInhibitedSource) -> routes.CronEntry: + path = "/etc/cron.daily/apport" + return routes.CronEntry(path, "regular", 1, 2, 3, 0o755, 0, cron.STOCK_INPUTS[path][1], True, inhibited_source=proof) + + +def _recovery_binding(entry: routes.CronEntry) -> tuple[routes.CronEntry, ...]: + return routes._cron_approval_binding((entry,), recovery=True) + + +@contextmanager +def _approved_cron_routes(monkeypatch: pytest.MonkeyPatch, entry: routes.CronEntry, profile: str = "recovery"): + from tests.test_management_containment_route_manifest import _closed_routes, _closed_v2_routes + + fixture_type = _closed_routes if profile == "v1" else _closed_v2_routes + with fixture_type(monkeypatch) as fixture: + fixture.facts["cron.service"] = routes.SystemdRouteFact("cron.service", "a" * 64, "masked", "inactive", 0, True) + fixture.catalog = tuple(sorted(fixture.facts)) + if profile == "recovery": + fixture.ids += ("f" * 64,) + fixture.docker = tuple(SimpleNamespace(container_id=item) for item in fixture.ids) + selected = fixture.approval.enrollment + scope = routes.RecoveryRouteScope(selected.deployment_id, selected.enrollment_id, selected.authorization_id, "ordinary-recovery-v1", fixture.ids, selected.original_files) + fixture.approval = routes.RecoveryRouteSelection(scope, fixture.approval.enrollment_sha256) + fixture.manifest = replace(fixture.manifest, schema="approved-management-recovery-routes-v1", profile=scope.profile, docker_container_ids=fixture.ids) + fixture.manifest = replace(fixture.manifest, cron_entries=(entry,), cron_restarter_paths=(entry.path,), systemd_routes=tuple(sorted((*fixture.manifest.systemd_routes, routes.SystemdRoute("cron.service", "a" * 64, "restarter")), key=lambda item: item.unit))) + fixture.save() + yield fixture + + @pytest.fixture def closed_proof(scope_tree) -> cron.CronInhibitedSource: properties = {name: "" for name in cron._PROPERTIES} @@ -89,6 +119,116 @@ def test_missing_cron_and_user_scopes_are_explicitly_bound(scope_tree) -> None: assert _scope(scope_tree) == first +@pytest.mark.parametrize("change", ("added", "removed", "recreated")) +def test_recovery_approval_allows_only_unrelated_directory_lifecycle(scope_tree, closed_proof, change: str) -> None: + root, _, _ = scope_tree + target = root / "system.slice" / ("docker-" + "a" * 64 + ".scope") + if change != "added": + _write_group(target, processes="1234\n", threads="1234\n", populated="1") + approved = _stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))) + if change != "added": + target.rename(root.parent / "retained-old-target-scope") + if change != "removed": + _write_group(target, processes="1234\n", threads="1234\n", populated="1") + current_scope = _scope(scope_tree) + assert _scope(scope_tree) == current_scope + current = _stock_entry(replace(closed_proof, cgroups=current_scope)) + assert current != approved + assert _recovery_binding(current) == _recovery_binding(approved) + assert routes._cron_approval_binding((current,), recovery=False) != routes._cron_approval_binding((approved,), recovery=False) + assert current.inhibited_source.cgroups == current_scope + + +@pytest.mark.parametrize("relative", ("", "system.slice", "system.slice/cron.service", "system.slice/cron.service/child", "other.slice", "other.slice/cron.service", "user.slice")) +def test_recovery_approval_retains_every_inhibited_directory_and_ancestor(scope_tree, closed_proof, relative: str) -> None: + root, _, _ = scope_tree + for path in ("system.slice/cron.service/child", "other.slice/cron.service", "user.slice"): + _write_group(root / path) + for path in ("system.slice/cron.service", "other.slice"): + _write_group(root / path) + approved = _stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))) + target = root / relative + target.chmod(stat.S_IMODE(target.stat().st_mode) ^ stat.S_IXGRP) + current = _stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))) + assert _recovery_binding(current) != _recovery_binding(approved) + + +@pytest.mark.parametrize("change", ("mount-id", "mount-device", "required-roots", "missing-roots", "empty-groups", "boot", "scheduler", "controller", "vendor", "mask", "inhibited-services", "cron-bytes", "cron-inode", "missing-proof")) +def test_recovery_approval_retains_other_scope_and_source_bindings(closed_proof, change: str) -> None: + approved = _stock_entry(closed_proof) + scope = closed_proof.cgroups + if change == "mount-id": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, mount_id=scope.mount_id + 1))) + elif change == "mount-device": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, mount_device="0:999"))) + elif change == "required-roots": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, required_empty_roots=("/other.slice/cron.service", *scope.required_empty_roots)))) + elif change == "missing-roots": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, missing_roots=()))) + elif change == "empty-groups": + current = _stock_entry(replace(closed_proof, cgroups=replace(scope, empty_groups=(cron.EmptyCgroupFact("/user.slice", (), (), 0, 0),)))) + elif change == "boot": + current = _stock_entry(replace(closed_proof, boot_id="00000000-0000-4000-8000-000000000001")) + elif change == "scheduler": + current = _stock_entry(replace(closed_proof, scheduler_sha256="b" * 64)) + elif change == "controller": + current = _stock_entry(replace(closed_proof, controller_properties=tuple((key, "system.slice" if key == "Slice" else value) for key, value in closed_proof.controller_properties))) + elif change == "vendor": + current = _stock_entry(replace(closed_proof, sources=(replace(closed_proof.sources[0], inode=99), *closed_proof.sources[1:]))) + elif change == "mask": + current = _stock_entry(replace(closed_proof, mask_identity=(1, 99, *closed_proof.mask_identity[2:]))) + elif change == "inhibited-services": + current = _stock_entry(replace(closed_proof, inhibited_services=(("cron.service", "b" * 64),))) + elif change == "cron-bytes": + current = replace(approved, sha256="f" * 64) + elif change == "cron-inode": + current = replace(approved, inode=99) + else: + current = replace(approved, inhibited_source=None) + assert _recovery_binding(current) != _recovery_binding(approved) + + +@pytest.mark.parametrize("profile", ("recovery", "v1", "v2")) +def test_only_recovery_route_reader_allows_unrelated_scope_and_keeps_full_facts(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, profile: str) -> None: + approved = _stock_entry(closed_proof) + with _approved_cron_routes(monkeypatch, approved, profile) as fixture: + original = fixture.path.read_bytes() + target = scope_tree[0] / "system.slice" / ("docker-" + "a" * 64 + ".scope") + _write_group(target, processes="1234\n", threads="1234\n", populated="1") + + def read_catalog(**kwargs): + return (_stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))),) + + monkeypatch.setattr(routes, "_cron_catalog", read_catalog) + if profile != "recovery": + with pytest.raises(routes.RouteManifestUnavailable, match="changed cron/at route"): + fixture.observe() + else: + observation = fixture.observe() + assert observation.cron_entries == read_catalog() + assert observation.cron_entries != observation.manifest.cron_entries + assert "/system.slice/" + target.name in {item.path for item in observation.cron_entries[0].inhibited_source.cgroups.directories} + assert fixture.path.read_bytes() == original + + +def test_recovery_fresh_route_repeat_still_refuses_unrelated_scope_change(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + approved = _stock_entry(closed_proof) + with _approved_cron_routes(monkeypatch, approved) as fixture: + calls = 0 + + def read_catalog(**kwargs): + nonlocal calls + calls += 1 + if calls == 2: + _write_group(scope_tree[0] / "system.slice/docker-new.scope") + return (_stock_entry(replace(closed_proof, cgroups=_scope(scope_tree))),) + + monkeypatch.setattr(routes, "_cron_catalog", read_catalog) + with pytest.raises(routes.RouteManifestUnavailable, match="installed ordinary routes changed during observation"): + fixture.observe() + assert calls == 2 + + @pytest.mark.parametrize("state", ("leader", "thread", "nested", "event-only", "user-session", "alternative-slice")) def test_masked_controller_cannot_hide_real_scope_population(scope_tree, state: str) -> None: root, _, _ = scope_tree From f56a85131774b37aee6e7490e1c9f1c461df48dd Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 04:40:34 +0900 Subject: [PATCH 19/24] fix(management): bind retained masked watcher sources --- ...gement-recovery-containment-observation.md | 17 +- scripts/recovery_epoch_containment_fixture.py | 2 +- .../management/recovery_containment.py | 11 +- .../management/recovery_containment_model.py | 7 +- .../management/recovery_watcher_source.py | 170 ++++++++++++ tests/test_management_recovery_containment.py | 8 +- ...nagement_recovery_containment_resources.py | 5 +- ...test_management_recovery_watcher_source.py | 244 ++++++++++++++++++ 8 files changed, 455 insertions(+), 9 deletions(-) create mode 100644 src/auto_coder/management/recovery_watcher_source.py create mode 100644 tests/test_management_recovery_watcher_source.py diff --git a/docs/client-features/management-recovery-containment-observation.md b/docs/client-features/management-recovery-containment-observation.md index ee478843b..02c14ea32 100644 --- a/docs/client-features/management-recovery-containment-observation.md +++ b/docs/client-features/management-recovery-containment-observation.md @@ -30,7 +30,7 @@ strict parsers are in `recovery_containment_model.py`. | Started original | Original boot ID; protected original manager/proxy lifetime capture; exact original owner file; live original namespace holder and its actual retained descriptor | | Never-started original | Existing Docker objects currently in the pristine `created` state; no invented owner, process lifetime, prior record or namespace pin | | Target | Exact separate manager/image/instance; protected bootstrap and management configuration; complete bind roots; prepared owner inode; distinct loopback published port | -| Watcher | Exact installed systemd unit, actual installed module and protected configuration, and the existing `proxy-watch.lock` that the real watcher takes before tick/reconcile | +| Watcher | Exact installed systemd unit and retained `unit_source` file identity, actual installed module and protected configuration, and the existing `proxy-watch.lock` that the real watcher takes before tick/reconcile | There are at most 16 original pairs and 32 watchers. Their complete eventual peer custody set must also satisfy `3 + 2 × started originals + watchers <= 64`; the @@ -111,6 +111,21 @@ unsupported remote origins cannot be converted into closure by an operator list. Each real watcher must use its enrolled source/configuration and actual existing controller lock. The observer changes no unit, restart policy, file or controller. +A masked effective unit has no executable command inventory. Recovery therefore +requires a separate original `unit_source: RecoveryFile` for each watcher. The +reader queries the actual system manager's `UnitPath`, verifies one protected +mask above the selected regular fragment, and refuses competing fragments, +aliases, or unit-specific, dash-prefix and type-wide drop-ins. The retained unit +must use the supported simple-service profile with `Restart=no` and exactly the +current observer's literal interpreter path followed by +`-I -B -m auto_coder.management.proxy_watch_cli --config `. +Another virtual environment sharing the Python binary is a different selection. +Environment, working-directory, root-directory and additional executable +directives are unsupported. The observer independently repeats the fragment and +load-path reads while retaining the existing actual mask, current task and +controller-lock requirements. Its non-authorizing watcher facts retain this +source evidence; no admission, dispatch or readiness event is added. + For each started original, the observer checks the exact captured manager/proxy lifetimes and mount identities, every remaining cgroup task, absence of captured escaped tasks, the whole host's task references to the retained original network diff --git a/scripts/recovery_epoch_containment_fixture.py b/scripts/recovery_epoch_containment_fixture.py index b11eeb544..b9b1913a1 100644 --- a/scripts/recovery_epoch_containment_fixture.py +++ b/scripts/recovery_epoch_containment_fixture.py @@ -120,7 +120,7 @@ def start_watcher(generation: Generation, original: RecoveryOriginal, *, first: """Run, observe and inhibit the real watcher before retaining its EX lock.""" unit = "auto-coder-management-proxy-watch" + ("" if first else "-" + generation.name) + ".service" fixture = WatcherFixture.start(generation.root, original.management_id, original.management_image_id, original.proxy_id, original.proxy_image_id, Path(original.proxy_config.path), unit=unit) - selected = RecoveryWatcher(unit, selected_file(fixture.config), selected_file(Path(proxy_watch_cli.__file__), source=True), selected_lock(fixture.state / "proxy-watch.lock")) + selected = RecoveryWatcher(unit, selected_file(fixture.config), selected_file(Path(proxy_watch_cli.__file__), source=True), selected_file(fixture.source, source=True), selected_lock(fixture.state / "proxy-watch.lock")) return fixture, selected diff --git a/src/auto_coder/management/recovery_containment.py b/src/auto_coder/management/recovery_containment.py index 19b1bfcf8..860f3e468 100644 --- a/src/auto_coder/management/recovery_containment.py +++ b/src/auto_coder/management/recovery_containment.py @@ -64,6 +64,7 @@ ) from .recovery_containment_profile import RecoveryManagerFact, _mapped_file, assert_disjoint_target, digest, read_bound_file, validate_manager_profile, validate_proxy_profile from .recovery_containment_resources import RecoveryResourceObservation, _private_lock, hold_observed_resources +from .recovery_watcher_source import RecoveryWatcherUnitFact, observe_recovery_watcher_unit logger = get_logger(__name__) _REPORT_SCHEMA = "current-recovery-containment-v1" @@ -112,6 +113,7 @@ class RecoveryWatcherFact: proxy_id: str configuration: RecoveryFile source: RecoveryFile + installed_unit: RecoveryWatcherUnitFact lock_device: int lock_inode: int @@ -199,7 +201,7 @@ def route_selection(approved: RecoveryEnrollmentObservation) -> RecoveryRouteSel originals.extend((OriginalFile("bootstrap-config", **asdict(item.bootstrap_config)), OriginalFile("management-config", **asdict(item.management_config)), OriginalFile("proxy-config", **asdict(item.proxy_config)))) originals.extend((OriginalFile("bootstrap-config", **asdict(selected.target.bootstrap_config)), OriginalFile("management-config", **asdict(selected.target.management_config)))) for watcher in selected.watchers: - originals.extend((OriginalFile("watcher-source", **asdict(watcher.source)), OriginalFile("watcher-config", **asdict(watcher.configuration)))) + originals.extend((OriginalFile("watcher-source", **asdict(watcher.source)), OriginalFile("watcher-config", **asdict(watcher.configuration)), OriginalFile("support-source", **asdict(watcher.unit_source)))) scope = RecoveryRouteScope(selected.deployment_id, selected.enrollment_id, selected.authorization_id, selected.profile, selected.container_ids, tuple(originals)) return RecoveryRouteSelection(scope, approved.sha256) @@ -320,9 +322,10 @@ def _watchers(enrollment: RecoveryEnrollment, routes: RouteManifestObservation) root = Path(raw["state_root"]) _approved_directory(root, uid=0) fact = routes_by_name.get(selected.unit) - if fact is None or not fact.inhibited or fact.main_pid != 0 or {selected.source.path, selected.configuration.path} - {item.path for item in fact.command_sources}: - raise RecoveryContainmentUnavailable("watcher installed command or technical inhibition differs") - result.append(RecoveryWatcherFact(selected.unit, binding.manager_id, binding.proxy_id, selected.configuration, selected.source, selected.lock.device, selected.lock.inode)) + if fact is None: + raise RecoveryContainmentUnavailable("watcher installed unit is outside the observed routes") + installed_unit = observe_recovery_watcher_unit(selected, fact) + result.append(RecoveryWatcherFact(selected.unit, binding.manager_id, binding.proxy_id, selected.configuration, selected.source, installed_unit, selected.lock.device, selected.lock.inode)) source_sets = [(item.unit, item.command_sources) for item in routes.systemd] source_sets += [(str(item.uid) + ":" + item.fact.unit, item.fact.command_sources) for item in routes.user_systemd] source_sets += [(item.path, item.command_sources) for item in routes.cron_entries] diff --git a/src/auto_coder/management/recovery_containment_model.py b/src/auto_coder/management/recovery_containment_model.py index 100145b5e..f3d7648f0 100644 --- a/src/auto_coder/management/recovery_containment_model.py +++ b/src/auto_coder/management/recovery_containment_model.py @@ -191,12 +191,17 @@ class RecoveryWatcher: unit: str configuration: RecoveryFile source: RecoveryFile + unit_source: RecoveryFile lock: RecoveryLock @classmethod def parse(cls, value: object) -> RecoveryWatcher: raw = _object(value, cls.__dataclass_fields__, "recovery watcher") - return cls(_text(raw["unit"], re.compile(r"[A-Za-z0-9_.@:-]{1,120}\.service"), "watcher unit"), RecoveryFile.parse(raw["configuration"]), RecoveryFile.parse(raw["source"]), RecoveryLock.parse(raw["lock"])) + unit = _text(raw["unit"], re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,119}\.service"), "watcher unit") + fragment = RecoveryFile.parse(raw["unit_source"]) + if Path(fragment.path).name != unit: + raise RecoveryContainmentUnavailable("recovery watcher retained unit source name differs") + return cls(unit, RecoveryFile.parse(raw["configuration"]), RecoveryFile.parse(raw["source"]), fragment, RecoveryLock.parse(raw["lock"])) @dataclass(frozen=True) diff --git a/src/auto_coder/management/recovery_watcher_source.py b/src/auto_coder/management/recovery_watcher_source.py new file mode 100644 index 000000000..04af9d647 --- /dev/null +++ b/src/auto_coder/management/recovery_watcher_source.py @@ -0,0 +1,170 @@ +"""Read the finite retained watcher unit behind its actual systemd mask. + +Masked effective units have no executable command to inspect. This recovery-only +profile independently binds the retained fragment selected from the real system +manager's load paths. It never changes a unit, mask, controller or approval. +""" + +from __future__ import annotations + +import os +import re +import stat +import sys +from dataclasses import dataclass +from pathlib import Path + +from .containment_enrollment import _trusted_ancestors +from .containment_route_manifest import SystemdRouteFact, _command +from .recovery_containment_model import RecoveryContainmentUnavailable, RecoveryFile, RecoveryWatcher +from .recovery_containment_profile import read_bound_file + +_UNIT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,119}\.service") +_LITERAL_PATH = re.compile(r"/[A-Za-z0-9_./:+@=-]+") +_MAX_ENTRIES = 16384 + + +@dataclass(frozen=True) +class RecoveryWatcherUnitFact: + source: RecoveryFile + interpreter: str + load_paths: tuple[str, ...] + load_roots: tuple[tuple[str, int, int, int, int, int], ...] + mask_path: str + mask_identity: tuple[int, ...] + + +def _load_paths() -> tuple[str, ...]: + raw = _command("systemctl", "show", "--property=UnitPath", "--value", max_bytes=16384) + paths = tuple(raw.strip().split()) + if not 1 <= len(paths) <= 32 or len(set(paths)) != len(paths) or any(not _LITERAL_PATH.fullmatch(path) or ".." in Path(path).parts or str(Path(path)) != path for path in paths): + raise RecoveryContainmentUnavailable("recovery watcher system unit load paths are unsupported") + return paths + + +def _drop_in_names(unit: str) -> tuple[str, ...]: + stem = unit.removesuffix(".service") + prefixes = [stem[: index + 1] + ".service.d" for index, character in enumerate(stem) if character == "-"] + return tuple(sorted({"service.d", unit + ".d", *prefixes})) + + +def _literal_command(content: bytes, selected: RecoveryWatcher) -> None: + if len(content) > 16384 or any(value < 32 and value not in (9, 10) or value >= 127 for value in content) or not _UNIT.fullmatch(selected.unit): + raise RecoveryContainmentUnavailable("recovery watcher retained unit is outside the finite profile") + expected = sys.executable + " -I -B -m auto_coder.management.proxy_watch_cli --config " + selected.configuration.path + if any(not _LITERAL_PATH.fullmatch(path) or ".." in Path(path).parts for path in (sys.executable, selected.configuration.path)): + raise RecoveryContainmentUnavailable("recovery watcher literal interpreter/configuration path is unsupported") + sections: dict[str, dict[str, str]] = {} + section = "" + for raw in content.decode("ascii").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line in ("[Unit]", "[Service]"): + section = line[1:-1] + if section in sections: + raise RecoveryContainmentUnavailable("recovery watcher retained unit repeats a section") + sections[section] = {} + continue + key, separator, value = line.partition("=") + allowed = {"Description"} if section == "Unit" else {"Type", "ExecStart", "Restart", "StandardOutput", "StandardError"} if section == "Service" else set() + if not separator or key not in allowed or key in sections[section] or not value or any(character in value for character in ("\\", "%", "\r", "\x00")): + raise RecoveryContainmentUnavailable("recovery watcher retained unit has unsupported directives") + sections[section][key] = value + service = sections.get("Service", {}) + if service.get("Type") != "simple" or service.get("Restart") != "no" or service.get("ExecStart") != expected: + raise RecoveryContainmentUnavailable("recovery watcher command does not bind this interpreter/module/configuration") + for name in ("StandardOutput", "StandardError"): + value = service.get(name, "journal") + if value not in ("journal", "null") and (not value.startswith("append:") or not _LITERAL_PATH.fullmatch(value.removeprefix("append:")) or ".." in Path(value.removeprefix("append:")).parts): + raise RecoveryContainmentUnavailable("recovery watcher retained output selection is unsupported") + + +def _scope(selected: RecoveryWatcher, *, expected_uid: int) -> RecoveryWatcherUnitFact: + paths = _load_paths() + source = Path(selected.unit_source.path) + if source.name != selected.unit or str(source.parent) not in paths: + raise RecoveryContainmentUnavailable("recovery watcher retained source is outside the actual unit load path") + roots = [] + masks: list[tuple[int, str, tuple[int, ...]]] = [] + fragments: list[tuple[int, str]] = [] + aliases: list[tuple[str, str]] = [] + entries = 0 + for priority, name in enumerate(paths): + root = Path(name) + if root.resolve(strict=False) != root: + raise RecoveryContainmentUnavailable("recovery watcher unit load path is redirected") + try: + info = root.lstat() + except FileNotFoundError: + ancestor = root.parent + while not ancestor.exists(): + ancestor = ancestor.parent + _trusted_ancestors(ancestor, uid=expected_uid) + roots.append((name, 0, 0, 0, 0, 0)) + continue + _trusted_ancestors(root, uid=expected_uid) + roots.append((name, info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid)) + for drop_in in _drop_in_names(selected.unit): + try: + (root / drop_in).lstat() + except FileNotFoundError: + continue + raise RecoveryContainmentUnavailable("recovery watcher retained unit has unsupported drop-ins") + candidate = root / selected.unit + try: + candidate_info = candidate.lstat() + except FileNotFoundError: + candidate_info = None + if candidate_info is not None: + if candidate_info.st_uid != expected_uid or candidate_info.st_nlink != 1: + raise RecoveryContainmentUnavailable("recovery watcher fragment/mask is not independently protected") + if stat.S_ISLNK(candidate_info.st_mode) and os.readlink(candidate) == "/dev/null": + identity = (candidate_info.st_dev, candidate_info.st_ino, candidate_info.st_ctime_ns, candidate_info.st_mode, candidate_info.st_uid, candidate_info.st_gid) + masks.append((priority, str(candidate), identity)) + elif stat.S_ISREG(candidate_info.st_mode) and candidate == source: + fragments.append((priority, str(candidate))) + else: + raise RecoveryContainmentUnavailable("recovery watcher retained fragment selection is ambiguous") + with os.scandir(root) as listing: + for entry in listing: + entries += 1 + if entries > _MAX_ENTRIES: + raise RecoveryContainmentUnavailable("recovery watcher unit alias inventory exceeds its bound") + if entry.name == selected.unit or not entry.name.endswith(".service") or not entry.is_symlink(): + continue + target = os.readlink(entry.path) + if not target or len(target) > 4096: + raise RecoveryContainmentUnavailable("recovery watcher unit alias target is unsupported") + # systemd resolves aliases inside UnitPath by unit name even + # when their literal target file is absent at that load level. + aliases.append((entry.name, Path(target).name)) + try: + resolved = Path(entry.path).resolve(strict=False) + except RuntimeError as error: + raise RecoveryContainmentUnavailable("recovery watcher unit alias inventory contains a loop") from error + if resolved == source: + raise RecoveryContainmentUnavailable("recovery watcher retained fragment has an unsupported alias") + names = {selected.unit} + for _ in range(len(aliases) + 1): + related = {name for name, target in aliases if target in names} + if related <= names: + break + names.update(related) + if names != {selected.unit}: + raise RecoveryContainmentUnavailable("recovery watcher retained unit name has an unsupported alias") + if len(masks) != 1 or len(fragments) != 1 or masks[0][0] >= fragments[0][0]: + raise RecoveryContainmentUnavailable("recovery watcher requires one real mask above its retained fragment") + return RecoveryWatcherUnitFact(selected.unit_source, sys.executable, paths, tuple(roots), masks[0][1], masks[0][2]) + + +def observe_recovery_watcher_unit(selected: RecoveryWatcher, fact: SystemdRouteFact, *, expected_uid: int = 0) -> RecoveryWatcherUnitFact: + """Repeat actual load-path/source reads; effective inhibition is separate.""" + if fact.unit != selected.unit or not fact.inhibited or fact.main_pid != 0 or fact.needs_daemon_reload: + raise RecoveryContainmentUnavailable("recovery watcher effective unit is not technically inhibited") + first = _scope(selected, expected_uid=expected_uid) + content = read_bound_file(selected.unit_source, source=True, expected_uid=expected_uid) + _literal_command(content, selected) + if _scope(selected, expected_uid=expected_uid) != first or read_bound_file(selected.unit_source, source=True, expected_uid=expected_uid) != content: + raise RecoveryContainmentUnavailable("recovery watcher retained unit source changed during observation") + return first diff --git a/tests/test_management_recovery_containment.py b/tests/test_management_recovery_containment.py index d6e2851ca..05cc0b4a9 100644 --- a/tests/test_management_recovery_containment.py +++ b/tests/test_management_recovery_containment.py @@ -81,7 +81,13 @@ def _enrollment(*, generations: int = 1, watchers: int = 1, started: bool = True 18081, ) controls = tuple( - RecoveryWatcher(f"auto-coder-management-proxy-watch-{index:02d}.service", _file(f"/private/watcher-{index}/config.json", 10000 + index), _file("/private/package/management/proxy_watch_cli.py", 20000), RecoveryLock(f"/private/watcher-{index}/state/proxy-watch.lock", 1, 30000 + index)) + RecoveryWatcher( + f"auto-coder-management-proxy-watch-{index:02d}.service", + _file(f"/private/watcher-{index}/config.json", 10000 + index), + _file("/private/package/management/proxy_watch_cli.py", 20000), + _file(f"/run/systemd/system/auto-coder-management-proxy-watch-{index:02d}.service", 40000 + index), + RecoveryLock(f"/private/watcher-{index}/state/proxy-watch.lock", 1, 30000 + index), + ) for index in range(watchers) ) return RecoveryEnrollment("approved-management-recovery-containment-v1", "deployment", "enrollment", "authorization", RECOVERY_PROFILE, "local-engine", "sha256:" + "c" * 64, tuple(_original(index, started=started) for index in range(generations)), target, controls, True) diff --git a/tests/test_management_recovery_containment_resources.py b/tests/test_management_recovery_containment_resources.py index d2e2cc5cc..dbbaa7657 100644 --- a/tests/test_management_recovery_containment_resources.py +++ b/tests/test_management_recovery_containment_resources.py @@ -211,7 +211,10 @@ def resource_case(mode: str = "independent") -> Iterator[ResourceCase]: configuration = bound(paths["configuration"]) original = RecoveryOriginal("generation-one", "original-one", "a" * 64, "sha256:" + "b" * 64, "c" * 64, "sha256:" + "d" * 64, "started", observed["boot"], configuration, configuration, configuration, (), selected_lock(paths["original"]), configuration, pin) target = RecoveryTarget("target-one", "e" * 64, "sha256:" + "f" * 64, configuration, configuration, (), selected_lock(paths["target"]), 18080) - watcher = RecoveryWatcher("recovery-watch.service", configuration, configuration, selected_lock(paths["controller"])) + unit = root / "recovery-watch.service" + unit.write_bytes(b"# Resource-observation fixture; no executable controller.\n") + unit.chmod(0o600) + watcher = RecoveryWatcher("recovery-watch.service", configuration, configuration, bound(unit), selected_lock(paths["controller"])) enrollment = RecoveryEnrollment("approved-management-recovery-containment-v1", "deployment-one", "enrollment-one", "authorization-one", "ordinary-recovery-v1", "engine-one", "sha256:" + "0" * 64, (original,), target, (watcher,), True) yield ResourceCase(root, process, enrollment, resources, namespace, paths, parent_fds) finally: diff --git a/tests/test_management_recovery_watcher_source.py b/tests/test_management_recovery_watcher_source.py new file mode 100644 index 000000000..3fa58c353 --- /dev/null +++ b/tests/test_management_recovery_watcher_source.py @@ -0,0 +1,244 @@ +"""Real-file component checks; native systemd acceptance remains a VM gate.""" + +from __future__ import annotations + +import hashlib +import os +import pwd +import sys +import tempfile +from dataclasses import asdict, dataclass, replace +from pathlib import Path + +import pytest + +from auto_coder.management import recovery_watcher_source as watcher +from auto_coder.management.containment_route_manifest import SystemdRouteFact +from auto_coder.management.recovery_containment_model import RecoveryContainmentUnavailable, RecoveryFile, RecoveryLock, RecoveryWatcher + +pytestmark = pytest.mark.usefixtures("_use_real_commands") + + +def _bound(path: Path) -> RecoveryFile: + info = path.stat() + return RecoveryFile(str(path), hashlib.sha256(path.read_bytes()).hexdigest(), info.st_dev, info.st_ino) + + +def _identity(path: Path) -> tuple[int, ...]: + info = path.lstat() + return info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns + + +@dataclass +class UnitFixture: + root: Path + paths: tuple[Path, ...] + mask: Path + source: Path + selected: RecoveryWatcher + fact: SystemdRouteFact + content: bytes + commands: list[tuple[str, ...]] + + def observe(self): + return watcher.observe_recovery_watcher_unit(self.selected, self.fact, expected_uid=os.geteuid()) + + def select_source(self, content: bytes) -> None: + self.source.write_bytes(content) + self.selected = replace(self.selected, unit_source=_bound(self.source)) + + +@pytest.fixture +def unit_fixture(monkeypatch: pytest.MonkeyPatch): + with tempfile.TemporaryDirectory(prefix="ac-watcher-source-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: + root = Path(directory) + roots = tuple(root / name for name in ("etc-units", "run-units", "vendor-units")) + for path in roots: + path.mkdir(mode=0o700) + paths = (*roots, root / "absent-parent" / "absent-units") + config, module, lock = root / "watcher.json", root / "proxy_watch_cli.py", root / "proxy-watch.lock" + for path in (config, module, lock): + path.write_bytes(b"{}\n") + path.chmod(0o600) + unit = "ac-watch-proxy.service" + source, mask = roots[1] / unit, roots[0] / unit + command = sys.executable + " -I -B -m auto_coder.management.proxy_watch_cli --config " + str(config) + content = ("[Unit]\nDescription=Retained original watcher\n[Service]\nType=simple\nExecStart=" + command + "\nRestart=no\nStandardOutput=append:" + str(root / "watcher-output.log") + "\nStandardError=journal\n").encode() + source.write_bytes(content) + source.chmod(0o600) + mask.symlink_to("/dev/null") + info = lock.stat() + selected = RecoveryWatcher(unit, _bound(config), _bound(module), _bound(source), RecoveryLock(str(lock), info.st_dev, info.st_ino)) + fact = SystemdRouteFact(unit, "a" * 64, "masked", "inactive", 0, True, True) + commands = [] + + def command_read(*args, **kwargs): + commands.append(args) + assert args == ("systemctl", "show", "--property=UnitPath", "--value") + return " ".join(str(path) for path in paths) + "\n" + + monkeypatch.setattr(watcher, "_command", command_read) + yield UnitFixture(root, paths, mask, source, selected, fact, content, commands) + + +def test_masked_watcher_joins_real_retained_fragment_without_effective_commands(unit_fixture) -> None: + fixture = unit_fixture + assert fixture.fact.command_sources == () + source_before, mask_before = _identity(fixture.source), _identity(fixture.mask) + result = fixture.observe() + assert result.source == fixture.selected.unit_source + assert result.interpreter == sys.executable + assert result.load_paths == tuple(str(path) for path in fixture.paths) + assert result.mask_path == str(fixture.mask) + assert result.load_roots[-1][1:] == (0, 0, 0, 0, 0) + assert fixture.source.read_bytes() == fixture.content + assert _identity(fixture.source) == source_before and _identity(fixture.mask) == mask_before + assert len(fixture.commands) == 2 + assert RecoveryWatcher.parse(asdict(fixture.selected)) == fixture.selected + + +@pytest.mark.parametrize("field,value", (("inhibited", False), ("main_pid", 12), ("needs_daemon_reload", True), ("unit", "other.service"))) +def test_retained_source_cannot_replace_actual_technical_inhibition(unit_fixture, field, value) -> None: + unit_fixture.fact = replace(unit_fixture.fact, **{field: value}) + with pytest.raises(RecoveryContainmentUnavailable, match="technically inhibited"): + unit_fixture.observe() + assert not unit_fixture.commands + + +@pytest.mark.parametrize("change", ("missing-isolation", "missing-bytecode-flag", "module", "config", "extra-argument", "environment", "working-directory", "root-directory", "pre-command", "post-command", "duplicate-command", "install-alias", "section-repeat", "specifier", "control")) +def test_retained_unit_requires_the_literal_finite_command_profile(unit_fixture, change: str) -> None: + content = unit_fixture.content.decode() + if change == "missing-isolation": + content = content.replace(" -I -B ", " -B ") + elif change == "missing-bytecode-flag": + content = content.replace(" -I -B ", " -I ") + elif change == "module": + content = content.replace("auto_coder.management.proxy_watch_cli", "other.module") + elif change == "config": + content = content.replace(unit_fixture.selected.configuration.path, str(unit_fixture.root / "different.json")) + elif change == "extra-argument": + content = content.replace("\nRestart=", " --other\nRestart=") + elif change == "duplicate-command": + content += "ExecStart=/usr/bin/true\n" + elif change == "install-alias": + content += "[Install]\nAlias=other.service\n" + elif change == "section-repeat": + content += "[Service]\n" + elif change == "specifier": + content = content.replace("Description=Retained", "Description=%n Retained") + elif change == "control": + content = content.replace("Type=simple", "Type=simple\r") + else: + directive = {"environment": "Environment=PYTHONPATH=/other", "working-directory": "WorkingDirectory=/other", "root-directory": "RootDirectory=/other", "pre-command": "ExecStartPre=/usr/bin/true", "post-command": "ExecStartPost=/usr/bin/true"}[change] + content += directive + "\n" + unit_fixture.select_source(content.encode()) + with pytest.raises(RecoveryContainmentUnavailable): + unit_fixture.observe() + + +def test_same_python_binary_in_another_venv_is_not_the_same_interpreter_selection(unit_fixture) -> None: + other = unit_fixture.root / "other-venv/bin/python" + other.parent.mkdir(parents=True) + other.symlink_to(sys.executable) + assert other.samefile(sys.executable) + unit_fixture.select_source(unit_fixture.content.replace(sys.executable.encode(), str(other).encode())) + with pytest.raises(RecoveryContainmentUnavailable, match="interpreter/module/configuration"): + unit_fixture.observe() + + +@pytest.mark.parametrize("change", ("bytes", "inode", "writable", "hardlink")) +def test_retained_source_must_be_the_protected_original_object(unit_fixture, change: str) -> None: + fixture = unit_fixture + if change == "bytes": + fixture.source.write_bytes(fixture.content + b"# changed\n") + elif change == "inode": + fixture.source.rename(fixture.root / "retained-old-fragment") + fixture.source.write_bytes(fixture.content) + fixture.source.chmod(0o600) + elif change == "writable": + fixture.source.chmod(0o666) + else: + os.link(fixture.source, fixture.root / "other-link") + with pytest.raises(RecoveryContainmentUnavailable): + fixture.observe() + + +@pytest.mark.parametrize("name", ("ac-watch-proxy.service.d", "ac-watch-.service.d", "ac-.service.d", "service.d")) +def test_unit_dash_prefix_and_type_wide_drop_ins_refuse(unit_fixture, name: str) -> None: + (unit_fixture.paths[2] / name).mkdir() + with pytest.raises(RecoveryContainmentUnavailable, match="drop-ins"): + unit_fixture.observe() + + +@pytest.mark.parametrize("kind", ("competing-fragment", "extra-mask", "missing-mask", "source-above-mask", "source-outside-load-path")) +def test_real_unit_load_path_selection_cannot_be_ambiguous(unit_fixture, monkeypatch: pytest.MonkeyPatch, kind: str) -> None: + fixture = unit_fixture + if kind == "competing-fragment": + (fixture.paths[2] / fixture.selected.unit).write_bytes(fixture.content) + elif kind == "extra-mask": + (fixture.paths[2] / fixture.selected.unit).symlink_to("/dev/null") + elif kind == "missing-mask": + fixture.mask.unlink() + elif kind == "source-above-mask": + monkeypatch.setattr(watcher, "_command", lambda *args, **kwargs: " ".join(str(path) for path in (fixture.paths[1], fixture.paths[0], *fixture.paths[2:]))) + else: + monkeypatch.setattr(watcher, "_command", lambda *args, **kwargs: " ".join(str(path) for path in (fixture.paths[0], *fixture.paths[2:]))) + with pytest.raises(RecoveryContainmentUnavailable): + fixture.observe() + + +@pytest.mark.parametrize("kind", ("direct", "missing-vendor-target", "name-chain", "relative")) +def test_aliases_follow_systemd_unit_names_even_when_literal_target_is_missing(unit_fixture, kind: str) -> None: + fixture = unit_fixture + alias = fixture.paths[0] / "generic.service" + if kind == "direct": + alias.symlink_to(fixture.source) + elif kind == "missing-vendor-target": + target = fixture.paths[2] / fixture.selected.unit + assert not target.exists() + alias.symlink_to(target) + elif kind == "name-chain": + alias.symlink_to(fixture.paths[2] / "middle.service") + (fixture.paths[1] / "middle.service").symlink_to(fixture.paths[2] / fixture.selected.unit) + else: + (fixture.paths[1] / "generic.service").symlink_to(fixture.selected.unit) + with pytest.raises(RecoveryContainmentUnavailable, match="alias"): + fixture.observe() + + +@pytest.mark.parametrize("change", ("mask", "alias", "source")) +def test_retained_unit_readback_repeats_source_and_load_path_authority(unit_fixture, monkeypatch: pytest.MonkeyPatch, change: str) -> None: + fixture = unit_fixture + real_read = watcher.read_bound_file + calls = 0 + + def changing_read(*args, **kwargs): + nonlocal calls + value = real_read(*args, **kwargs) + calls += 1 + if calls == 1: + if change == "mask": + fixture.mask.rename(fixture.root / "old-mask") + fixture.mask.symlink_to("/dev/null") + elif change == "alias": + (fixture.paths[0] / "new-alias.service").symlink_to(fixture.paths[2] / fixture.selected.unit) + else: + fixture.source.write_bytes(fixture.content + b"# changed after first read\n") + return value + + monkeypatch.setattr(watcher, "read_bound_file", changing_read) + with pytest.raises(RecoveryContainmentUnavailable): + fixture.observe() + + +@pytest.mark.parametrize("change", ("missing-source", "wrong-name", "instance")) +def test_enrollment_requires_an_unambiguous_retained_watcher_fragment(unit_fixture, change: str) -> None: + raw = asdict(unit_fixture.selected) + if change == "missing-source": + raw.pop("unit_source") + elif change == "wrong-name": + raw["unit_source"]["path"] = str(unit_fixture.source.parent / "other.service") + else: + raw["unit"] = "ac-watch@one.service" + with pytest.raises(RecoveryContainmentUnavailable): + RecoveryWatcher.parse(raw) From 87c41a74e3d96037b3713711614e119556411b37 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 05:22:36 +0900 Subject: [PATCH 20/24] fix(test): retain causal containment refusal diagnostics --- ...gement-recovery-containment-observation.md | 8 + scripts/recovery_epoch_native_negatives.py | 13 +- .../test_management_recovery_epoch_host.py | 4 +- .../management/recovery_containment.py | 12 +- .../management/recovery_diagnostics.py | 53 +++++++ src/auto_coder/management/recovery_epoch.py | 26 +--- ...t_management_recovery_epoch_diagnostics.py | 143 ++++++++++++++++++ tests/test_management_recovery_epoch_host.py | 45 ++++++ ...agement_recovery_epoch_native_negatives.py | 18 +++ 9 files changed, 292 insertions(+), 30 deletions(-) create mode 100644 src/auto_coder/management/recovery_diagnostics.py diff --git a/docs/client-features/management-recovery-containment-observation.md b/docs/client-features/management-recovery-containment-observation.md index 02c14ea32..95a545d1d 100644 --- a/docs/client-features/management-recovery-containment-observation.md +++ b/docs/client-features/management-recovery-containment-observation.md @@ -178,6 +178,14 @@ cannot create a lock, pin or approval. Reports expose hashes and identities rath than configuration contents. Logger diagnostics may also appear on stdout, so a machine caller selects the structured JSON result line. +Unavailable CLI results and Loguru stdout/file diagnostics also retain at most +eight explicit exception causes, limited to type, integer error number and safe +code basename/function/line. They omit inner messages, arguments, full paths and +frame locals. The native fixture preserves validated causes in its separate +`containment_failure` result field, so truncating ordinary diagnostic log lines +cannot erase the causal locations. This metadata never changes a refusal, +acceptance predicate, report authority or exit status. + The separate verifier repeats real source, kernel, controller and holder reads. It rejects changed facts, malformed or authorizing reports, invalid digests, foreign boots, future samples and samples older than ten minutes. The earlier diff --git a/scripts/recovery_epoch_native_negatives.py b/scripts/recovery_epoch_native_negatives.py index 5f4f62db3..86c20a336 100644 --- a/scripts/recovery_epoch_native_negatives.py +++ b/scripts/recovery_epoch_native_negatives.py @@ -39,6 +39,7 @@ from auto_coder.management.original_attempts import canonical from auto_coder.management.recovery_containment_model import RECOVERY_ENROLLMENT_ROOT, NamedDescriptor, RecoveryEnrollment, RecoveryLock, RecoveryOriginal, RecoveryResources from auto_coder.management.recovery_containment_profile import digest +from auto_coder.management.recovery_diagnostics import safe_refusal_causes from auto_coder.management.recovery_epoch_peer import inspect_kernel_lock from auto_coder.management.recovery_epoch_state import EPOCH_ROOT @@ -91,6 +92,14 @@ class EpochAlreadyPublished(AssertionError): """The fresh fixture has become uncertain and must never be reused.""" +class NativeContainmentFailure(AssertionError): + """Carry only bounded non-authorizing CLI metadata to the failure artifact.""" + + def __init__(self, message: str, *, action: str, returncode: int, causes: object): + super().__init__(message) + self.diagnostics = {"schema": "native-recovery-containment-cli-failure-v1", "action": action, "returncode": returncode, "causes": safe_refusal_causes(causes)} + + def assert_unreserved_epoch(root: Path) -> None: """Reject every record, head, partial name or unexpected epoch object.""" if not root.is_absolute() or root != root.resolve(strict=True): @@ -750,9 +759,9 @@ def _containment_cli(directory: Path, label: str, enrollment: str, resources: Pa raise AssertionError("the actual containment CLI returned an incomplete or authorizing result") if refusal is None: if completed.returncode != 0 or value.get("status") != "observed-current-recovery-containment": - raise AssertionError("the genuine native containment positive control was unavailable: " + str(value.get("reason"))) + raise NativeContainmentFailure("the genuine native containment positive control was unavailable: " + str(value.get("reason")), action="verify" if previous is not None else "observe", returncode=completed.returncode, causes=value.get("causes")) elif completed.returncode != 2 or value.get("status") != "unavailable" or refusal not in str(value.get("reason")): - raise AssertionError("the actual containment verifier did not reject the injected condition: " + label + " " + str(value.get("reason"))) + raise NativeContainmentFailure("the actual containment verifier did not reject the injected condition: " + label + " " + str(value.get("reason")), action="verify" if previous is not None else "observe", returncode=completed.returncode, causes=value.get("causes")) return value diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index efa4a5f57..626d60ab7 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -31,7 +31,7 @@ from scripts.recovery_epoch_containment_fixture import CapturedGeneration, capture_generation, enroll_recovery_scope, selected_file, start_watcher, target_from_container from scripts.recovery_epoch_fault_tracer import BOUNDARY_SCENARIOS, PUBLICATIONS, KernelPublicationBarrier, PublicationObservation, parse_boundary from scripts.recovery_epoch_mutation_control import CONTROL, CONTROL_SCHEMA, ORACLE, PostReceiptGrantDispatched, assert_post_receipt_refusal, validate_control_result, verify_source_evidence -from scripts.recovery_epoch_native_negatives import REQUIRED_NEGATIVE_CASES, REQUIRED_REPORT_CASES, NativeNegativeContext, _replacement, assert_unreserved_epoch, run_native_negatives, verify_native_containment_reports +from scripts.recovery_epoch_native_negatives import REQUIRED_NEGATIVE_CASES, REQUIRED_REPORT_CASES, NativeContainmentFailure, NativeNegativeContext, _replacement, assert_unreserved_epoch, run_native_negatives, verify_native_containment_reports from scripts.recovery_epoch_native_peer import LATE_MUTATION_SCENARIOS, POST_RECEIPT_MUTATION_SCENARIOS, REQUIRED_PEER_CASES, mutate_after_publication, observe_post_receipt_dispatch, observe_queued_grant, pause_target_before_grant, run_native_peer_cases from scripts.test_management_containment_closed_host import InhibitedStockRoutes, WatcherFixture, require_disposable_host from scripts.test_management_containment_enrollment_host import DisposableSystemdRoutes, _private_directory, _write_private @@ -1329,6 +1329,8 @@ def preservation() -> object: except BaseException as error: result["status"] = "failed" result["failure_type"] = type(error).__name__ + if isinstance(error, NativeContainmentFailure): + result["containment_failure"] = error.diagnostics _record_failure_sidecars(result, chain) _record_failure_catalog(result, target.enrollment_id if target is not None else None) command_failure = _command_failure(error) diff --git a/src/auto_coder/management/recovery_containment.py b/src/auto_coder/management/recovery_containment.py index 860f3e468..bedb1742b 100644 --- a/src/auto_coder/management/recovery_containment.py +++ b/src/auto_coder/management/recovery_containment.py @@ -16,6 +16,7 @@ import sqlite3 import stat import subprocess +import sys import tarfile import time import uuid @@ -24,7 +25,7 @@ from pathlib import Path from typing import cast -from auto_coder.logger_config import get_logger +from auto_coder.logger_config import get_logger, setup_logger from .archived_exclusions import unique_pairs from .containment_enrollment import OriginalFile, _approved_directory, _file_identity, _protected, _trusted_ancestors @@ -64,6 +65,7 @@ ) from .recovery_containment_profile import RecoveryManagerFact, _mapped_file, assert_disjoint_target, digest, read_bound_file, validate_manager_profile, validate_proxy_profile from .recovery_containment_resources import RecoveryResourceObservation, _private_lock, hold_observed_resources +from .recovery_diagnostics import refusal_causes from .recovery_watcher_source import RecoveryWatcherUnitFact, observe_recovery_watcher_unit logger = get_logger(__name__) @@ -555,16 +557,22 @@ def main() -> int: parser.add_argument("--resources", required=True, type=Path) parser.add_argument("--previous", type=Path) args = parser.parse_args() + setup_logger(stream=sys.stdout, log_file="/var/log/auto-coder/recovery-containment.log") try: resources = RecoveryResources.parse(_input(args.resources)) if (args.action == "verify") != (args.previous is not None): raise RecoveryContainmentUnavailable("separate verification requires exactly one previous report") report = verify(args.enrollment, _input(args.previous), resources) if args.previous is not None else observe(args.enrollment, resources) + logger.complete() print(json.dumps(asdict(report), sort_keys=True, separators=(",", ":")), flush=True) return 0 except (OSError, ValueError, TypeError, KeyError, IndexError, UnicodeError, subprocess.SubprocessError) as error: logger.bind(observation_only=True, recovery_grant=False, admission_created=False).warning("Current recovery containment unavailable: {}", str(error)) - print(json.dumps({"schema": _REPORT_SCHEMA, "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": str(error)}, sort_keys=True), flush=True) + causes = refusal_causes(error) + for index, cause in enumerate(causes, start=1): + logger.bind(observation_only=True, recovery_grant=False, admission_created=False).warning("Current recovery containment refusal cause {}: {}", index, json.dumps(cause, sort_keys=True, separators=(",", ":"))) + logger.complete() + print(json.dumps({"schema": _REPORT_SCHEMA, "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": str(error), "causes": causes}, sort_keys=True), flush=True) return 2 diff --git a/src/auto_coder/management/recovery_diagnostics.py b/src/auto_coder/management/recovery_diagnostics.py new file mode 100644 index 000000000..0117f942b --- /dev/null +++ b/src/auto_coder/management/recovery_diagnostics.py @@ -0,0 +1,53 @@ +"""Bounded non-authorizing code locations for recovery refusal diagnostics.""" + +from __future__ import annotations + +import re +from pathlib import Path + +_SYMBOL = r"[A-Za-z_][A-Za-z0-9_]{0,95}" +_FILENAME = r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}\.py" +_FUNCTION = _SYMBOL + r"|" + + +def refusal_causes(error: BaseException) -> list[dict[str, str | int]]: + """Describe bounded code locations, never exception data or frame locals.""" + causes: list[dict[str, str | int]] = [] + visited: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in visited and len(causes) < 8: + visited.add(id(current)) + name = type(current).__name__ + row: dict[str, str | int] = {"type": name if re.fullmatch(_SYMBOL, name) else "Exception"} + if isinstance(current, OSError) and type(current.errno) is int: + row["errno"] = current.errno + trace = current.__traceback__ + if trace is not None: + while trace.tb_next is not None: + trace = trace.tb_next + code = trace.tb_frame.f_code + filename = Path(code.co_filename).name + if re.fullmatch(_FILENAME, filename) and re.fullmatch(_FUNCTION, code.co_name): + row.update(file=filename, function=code.co_name, line=trace.tb_lineno) + causes.append(row) + current = current.__cause__ + return causes + + +def safe_refusal_causes(value: object) -> list[dict[str, str | int]]: + """Copy only the same finite metadata from a subprocess's JSON result.""" + if not isinstance(value, list) or len(value) > 8: + return [] + causes: list[dict[str, str | int]] = [] + for row in value: + if not isinstance(row, dict) or not set(row) <= {"type", "errno", "file", "function", "line"}: + return [] + if not isinstance(row.get("type"), str) or re.fullmatch(_SYMBOL, row["type"]) is None: + return [] + if "errno" in row and type(row["errno"]) is not int: + return [] + location = {"file", "function", "line"} & row.keys() + if location and (location != {"file", "function", "line"} or not isinstance(row["file"], str) or re.fullmatch(_FILENAME, row["file"]) is None or not isinstance(row["function"], str) or re.fullmatch(_FUNCTION, row["function"]) is None or type(row["line"]) is not int or row["line"] <= 0): + return [] + causes.append(dict(row)) + return causes diff --git a/src/auto_coder/management/recovery_epoch.py b/src/auto_coder/management/recovery_epoch.py index ef1d7090d..262cf96ad 100644 --- a/src/auto_coder/management/recovery_epoch.py +++ b/src/auto_coder/management/recovery_epoch.py @@ -11,7 +11,6 @@ import hashlib import json import os -import re import signal import socket import stat @@ -34,6 +33,7 @@ from .recovery_containment import observe as observe_containment from .recovery_containment import verify as verify_containment from .recovery_containment_model import RecoveryEnrollment, RecoveryEnrollmentObservation +from .recovery_diagnostics import refusal_causes as _refusal_causes from .recovery_epoch_authorization import ( PEER_ROOT, PEER_SOCKET, @@ -361,30 +361,6 @@ def run_issuer() -> EpochDispatch: signal.signal(saved_signal, handler) # type: ignore[arg-type] -def _refusal_causes(error: BaseException) -> list[dict[str, str | int]]: - """Describe bounded code locations, never exception data or frame locals.""" - causes: list[dict[str, str | int]] = [] - visited: set[int] = set() - current: BaseException | None = error - while current is not None and id(current) not in visited and len(causes) < 8: - visited.add(id(current)) - name = type(current).__name__ - row: dict[str, str | int] = {"type": name if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,95}", name) else "Exception"} - if isinstance(current, OSError) and type(current.errno) is int: - row["errno"] = current.errno - trace = current.__traceback__ - if trace is not None: - while trace.tb_next is not None: - trace = trace.tb_next - code = trace.tb_frame.f_code - filename = Path(code.co_filename).name - if re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}\.py", filename) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,95}|", code.co_name): - row.update(file=filename, function=code.co_name, line=trace.tb_lineno) - causes.append(row) - current = current.__cause__ - return causes - - def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Issue one approved management ownership epoch from fixed host authority") parser.parse_args(argv) diff --git a/tests/test_management_recovery_epoch_diagnostics.py b/tests/test_management_recovery_epoch_diagnostics.py index 6b96e5477..f999e41f4 100644 --- a/tests/test_management_recovery_epoch_diagnostics.py +++ b/tests/test_management_recovery_epoch_diagnostics.py @@ -4,13 +4,20 @@ import errno import json +import os import re +import subprocess +import sys +from dataclasses import asdict from pathlib import Path import pytest from auto_coder.logger_config import setup_logger +from auto_coder.management import recovery_containment as containment +from auto_coder.management import recovery_diagnostics as diagnostics from auto_coder.management import recovery_epoch as issuer +from auto_coder.management.recovery_containment_model import RecoveryContainmentUnavailable, RecoveryResources def _failure() -> ValueError: @@ -29,6 +36,7 @@ def _failure() -> ValueError: def test_refusal_causes_retain_only_code_identity_and_integer_errno() -> None: + assert issuer._refusal_causes is diagnostics.refusal_causes error = _failure() rows = issuer._refusal_causes(error) assert rows[0] == {"type": "ValueError", "file": Path(__file__).name, "function": "_failure", "line": error.__traceback__.tb_lineno} @@ -97,3 +105,138 @@ def refuse() -> None: assert rows[1] == {"type": "FileNotFoundError", "errno": errno.ENOENT, "file": "observed_component.py", "function": "fail", "line": 3} assert "private" not in output issuer.logger.remove() + + +@pytest.mark.parametrize("action", ["observe", "verify"]) +def test_containment_cli_preserves_masked_refusal_and_exports_safe_nested_causes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], action: str) -> None: + log = tmp_path / "containment.log" + resources = tmp_path / "resources.json" + resources.write_text(json.dumps(asdict(RecoveryResources(42, "12345", "00000000-0000-4000-8000-000000000001", (), (), ())))) + arguments = ["recovery_containment", action, "--enrollment", "component-observation", "--resources", str(resources)] + if action == "verify": + previous = tmp_path / "previous.json" + previous.write_text("{}") + arguments.extend(("--previous", str(previous))) + + def configure(**kwargs: object) -> None: + kwargs["log_file"] = str(log) + setup_logger(**kwargs) + + def masked(*_args: object) -> None: + try: + raise _failure() + except ValueError as cause: + raise RecoveryContainmentUnavailable("the original namespace pin cannot be retained") from cause + + monkeypatch.setattr(sys, "argv", arguments) + monkeypatch.setattr(containment, "setup_logger", configure) + monkeypatch.setattr(containment, action, masked) + assert containment.main() == 2 + containment.logger.complete() + standard = re.sub(r"\x1b\[[0-9;]*m", "", capsys.readouterr().out) + result = json.loads(standard.splitlines()[-1]) + assert result["schema"] == "current-recovery-containment-v1" and result["status"] == "unavailable" + assert result["reason"] == "the original namespace pin cannot be retained" + assert all(result[name] is False for name in ("grant_issued", "history_asserted", "admission_created")) + assert len(result["causes"]) == 3 + assert result["causes"][-1] == {"type": "FileNotFoundError", "errno": errno.ENOENT, "file": "observed_component.py", "function": "fail", "line": 3} + assert diagnostics.safe_refusal_causes(result["causes"]) == result["causes"] + for output in (standard, log.read_text()): + assert "Current recovery containment unavailable: the original namespace pin cannot be retained" in output + messages = [line.partition("Current recovery containment refusal cause ")[2] for line in output.splitlines() if "Current recovery containment refusal cause " in line] + assert [json.loads(line.partition(": ")[2]) for line in messages] == result["causes"] + assert "private" not in output and "fixed issuer refusal" not in output + containment.logger.remove() + + +def test_safe_subprocess_causes_are_a_detached_copy_of_the_exact_metadata() -> None: + rows = diagnostics.refusal_causes(_failure()) + parsed = diagnostics.safe_refusal_causes(json.loads(json.dumps(rows))) + assert parsed == rows + parsed[-1]["line"] = 100 + assert rows[-1]["line"] == 3 + + +@pytest.mark.parametrize( + "value", + [ + None, + {}, + "private-data", + [{"type": "ValueError"}] * 9, + ["private-data"], + [{"type": "ValueError", "message": "private-data"}], + [{"type": "invalid type"}], + [{"type": "ValueError", "errno": True}], + [{"type": "ValueError", "errno": "private-data"}], + [{"type": "ValueError", "file": "/private/observed.py", "function": "fail", "line": 3}], + [{"type": "ValueError", "file": "observed.py", "function": "private argument", "line": 3}], + [{"type": "ValueError", "file": "observed.py", "function": "fail", "line": True}], + [{"type": "ValueError", "file": "observed.py", "function": "fail", "line": 0}], + [{"type": "ValueError", "file": "observed.py"}], + [{"type": "V" * 97}], + ], +) +def test_subprocess_causes_reject_oversized_or_nonmetadata_fields(value: object) -> None: + assert diagnostics.safe_refusal_causes(value) == [] + + +@pytest.mark.usefixtures("_use_real_commands") +@pytest.mark.parametrize("unavailable", [False, True]) +def test_containment_cli_keeps_json_last_with_real_asynchronous_logging(tmp_path: Path, unavailable: bool) -> None: + resources = tmp_path / "resources.json" + resources.write_text(json.dumps(asdict(RecoveryResources(42, "12345", "00000000-0000-4000-8000-000000000001", (), (), ())))) + log = tmp_path / "queued.log" + program = r""" +import sys +import time +from dataclasses import dataclass +from auto_coder.logger_config import setup_logger +from auto_coder.management import recovery_containment as cli + +resources, logfile, unavailable = sys.argv[1:] + +class DelayedStandardOutput: + def write(self, text): + time.sleep(0.002) + return sys.stdout.write(text) + def flush(self): + return sys.stdout.flush() + +def configure(**kwargs): + setup_logger(stream=DelayedStandardOutput(), log_file=logfile) + assert all(handler._enqueue for handler in cli.logger._core.handlers.values()) + +@dataclass +class ComponentReport: + status: str = "observed-current-recovery-containment" + grant_issued: bool = False + history_asserted: bool = False + admission_created: bool = False + +def observe(*args): + for index in range(100): + cli.logger.warning("Queued component observation {}", index) + if unavailable == "yes": + try: + raise FileNotFoundError(2, "private-inner-message", "/private/source-token") + except OSError as cause: + raise cli.RecoveryContainmentUnavailable("fixed masked refusal") from cause + return ComponentReport() + +cli.setup_logger = configure +cli.observe = observe +sys.argv = ["recovery_containment", "observe", "--enrollment", "component", "--resources", resources] +raise SystemExit(cli.main()) +""" + environment = {name: value for name, value in os.environ.items() if name not in {"PYTEST_CURRENT_TEST", "PYTEST_RUNNER", "_PYTEST_CURRENT_TEST", "LLM_LOGGING_DISABLED"}} + environment["AUTO_CODER_LOG_FILE"] = str(log) + completed = subprocess.run([sys.executable, "-c", program, str(resources), str(log), "yes" if unavailable else "no"], capture_output=True, text=True, timeout=30, env=environment) + assert completed.returncode == (2 if unavailable else 0), completed.stderr + result = json.loads(completed.stdout.splitlines()[-1]) + assert result["status"] == ("unavailable" if unavailable else "observed-current-recovery-containment") + assert all(result[name] is False for name in ("grant_issued", "history_asserted", "admission_created")) + assert "Queued component observation 99" in completed.stdout and "Queued component observation 99" in log.read_text() + assert "private" not in completed.stdout and "private" not in log.read_text() + if unavailable: + assert result["causes"][-1] == {"type": "FileNotFoundError", "errno": 2} diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index 116512716..5b3be5d68 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -15,6 +15,7 @@ from pathlib import Path import pytest +from scripts import recovery_epoch_native_negatives as negatives from scripts import test_management_recovery_epoch_host as fixture from auto_coder.management.original_attempts import canonical @@ -109,6 +110,50 @@ def test_command_failure_exports_bounded_diagnostics_without_inputs_or_credentia assert fixture._command_failure(ValueError("unrelated error")) is None +def test_cli_failure_causes_survive_the_actual_final_result_log_truncation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + inputs = _inputs(tmp_path, monkeypatch) + root = tmp_path / "fixtures" / "original" + for name, path in (("FIXTURE_ROOT", root.parent), ("EPOCH_ROOT", tmp_path / "epoch"), ("PEER_ROOT", tmp_path / "peer"), ("AUTHORIZATION_ROOT", tmp_path / "authorization")): + monkeypatch.setattr(fixture, name, path) + monkeypatch.setattr(fixture, "_fixture_directory", lambda: root) + monkeypatch.setattr(fixture, "require_disposable_host", lambda _: {}) + + def private_directory(path: Path) -> None: + if path not in (Path("/var/lib/auto-coder"), Path("/etc/auto-coder")): + path.mkdir(mode=0o700) + + causes = [{"type": "OSError", "errno": 5, "file": "observed_component.py", "function": "read_current", "line": 42}] + document = {"schema": "current-recovery-containment-v1", "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": "fixed masked refusal", "causes": causes, "unrelated": "private-subprocess-data"} + monkeypatch.setattr(fixture, "_private_directory", private_directory) + monkeypatch.setattr(negatives.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 2, json.dumps(document) + "\n", "")) + + def refuse() -> None: + negatives._containment_cli(root, "original-observer", "component-observation", tmp_path / "private-resources.json") + + def many_later_diagnostics(_result: object, _chain: object) -> None: + (root / "issuer.log").write_text("RuntimeError: observed_component.py read_current line42\n" + "".join("RuntimeError: Later issuer diagnostic " + str(index) + "\n" for index in range(150))) + for index in range(150): + fixture.logger.error("Later bounded native diagnostic {}", index) + + monkeypatch.setattr(fixture.InhibitedStockRoutes, "install", refuse) + monkeypatch.setattr(fixture, "_record_failure_sidecars", many_later_diagnostics) + previous_umask = os.umask(0o077) + try: + with pytest.raises(negatives.NativeContainmentFailure, match="fixed masked refusal"): + fixture.run_fixture(**inputs) + fixture.logger.complete() + result = json.loads(inputs["output"].read_bytes()) + finally: + os.umask(previous_umask) + fixture.logger.remove() + assert result["status"] == "failed" and result["failure_type"] == "NativeContainmentFailure" + assert result["containment_failure"] == {"schema": "native-recovery-containment-cli-failure-v1", "action": "observe", "returncode": 2, "causes": causes} + assert 1 <= len(result["diagnostics"]) <= 120 and any("Later issuer diagnostic 149" in line for line in result["diagnostics"]) + assert "observed_component.py" not in "\n".join(result["diagnostics"]) + assert all(result[name] is False for name in ("no_child_started", "preserved_originals", "preserved_workers")) + assert "private-subprocess-data" not in json.dumps(result) and "private-resources.json" not in json.dumps(result) + + def _diagnostic_generation(root: Path, name: str = "A") -> Generation: generation = Generation(name, root / name, "component-container", []) generation.root.mkdir(parents=True) diff --git a/tests/test_management_recovery_epoch_native_negatives.py b/tests/test_management_recovery_epoch_native_negatives.py index 2fffbd876..20e8d8dae 100644 --- a/tests/test_management_recovery_epoch_native_negatives.py +++ b/tests/test_management_recovery_epoch_native_negatives.py @@ -48,6 +48,24 @@ def _context(tmp_path: Path) -> fixture.NativeNegativeContext: return fixture.NativeNegativeContext(_enrollment(), tmp_path, epoch) +@pytest.mark.parametrize("previous,refusal", [(False, None), (True, None), (True, "expected distinct refusal")]) +def test_containment_cli_propagates_only_safe_failure_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, previous: bool, refusal: str | None) -> None: + causes = [{"type": "OSError", "errno": 5, "file": "observed_component.py", "function": "read_current", "line": 42}] + document = {"schema": "current-recovery-containment-v1", "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": "fixed masked refusal", "causes": causes, "unrelated": "private-data"} + monkeypatch.setattr(fixture.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 2, "diagnostic prefix\n" + json.dumps(document) + "\n", "")) + with pytest.raises(fixture.NativeContainmentFailure, match="fixed masked refusal") as caught: + fixture._containment_cli(tmp_path, "original-observer", "component-observation", tmp_path / "private-resources.json", previous=tmp_path / "private-previous.json" if previous else None, refusal=refusal) + assert caught.value.diagnostics == {"schema": "native-recovery-containment-cli-failure-v1", "action": "verify" if previous else "observe", "returncode": 2, "causes": causes} + assert "private" not in json.dumps(caught.value.diagnostics) + assert (tmp_path / "original-observer.log").is_file() + + +def test_containment_cli_diagnostics_do_not_change_expected_refusal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + document = {"schema": "current-recovery-containment-v1", "status": "unavailable", "grant_issued": False, "history_asserted": False, "admission_created": False, "reason": "expected refusal", "causes": [{"type": "ValueError", "message": "private-data"}]} + monkeypatch.setattr(fixture.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 2, json.dumps(document) + "\n", "")) + assert fixture._containment_cli(tmp_path, "expected-refusal", "component-observation", tmp_path / "resources.json", refusal="expected refusal") == document + + def test_unreserved_guard_observes_but_never_rewrites_original_inputs(tmp_path: Path) -> None: context = _context(tmp_path) before = {path.name: fixture._projection(path) for path in context.epoch_root.iterdir()} From fd062abf6582ee0e0da1dde6156b60966ccbde91 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 05:28:11 +0900 Subject: [PATCH 21/24] fix(test): start readiness budget after verified child launch --- ...t_management_archived_cutover_container.py | 49 +++++- ...t_management_archived_cutover_readiness.py | 141 ++++++++++++++++++ 2 files changed, 185 insertions(+), 5 deletions(-) diff --git a/tests/test_management_archived_cutover_container.py b/tests/test_management_archived_cutover_container.py index b5899cdd4..dc422b96d 100644 --- a/tests/test_management_archived_cutover_container.py +++ b/tests/test_management_archived_cutover_container.py @@ -533,11 +533,47 @@ def call(self, generation: Generation, tool: str, arguments: dict[str, object], return {"http_status": error.code} raise ValueError("synthetic authenticated proxy HTTP unavailable") from None - def fresh_status(self, generation: Generation) -> dict[str, object]: - code = "import json;from pathlib import Path;from auto_coder.management.fresh_initialization import read_fresh_status;print(json.dumps(read_fresh_status(Path('/management-child-state'))))" + def fresh_status(self, generation: Generation, *, require_live: bool = False) -> dict[str, object]: + code = "import json;from pathlib import Path;from auto_coder.management.fresh_initialization import read_fresh_status;status=read_fresh_status(Path('/management-child-state'))\n" + if require_live: + code += ( + "import time\n" + "from auto_coder.management.bootstrap_supervisor import BootstrapConfig,ManagementBootstrapSupervisor\n" + "from auto_coder.management.self_update_handoff import observe_supervisor\n" + "config=BootstrapConfig.read(Path('/bootstrap/config.json'))\n" + "observed=observe_supervisor(config)\n" + "if status.get('status') not in {'launched','succeeded'} or observed is None:\n" + " raise ValueError('fresh launch is not live')\n" + "owner,child=observed\n" + "if child.phase.value!='running' or owner!=status['owner_incarnation'] or child.supervisor_incarnation!=owner or ManagementBootstrapSupervisor._identity(child)!=status['child_identity']:\n" + " raise ValueError('fresh launch lifetime differs')\n" + "if not status['started']<=time.time()<=status['deadline'] or not 0 None: + def wait_for_fresh_launch(self, generation: Generation) -> dict[str, object]: + # Source verification precedes dispatch. Give that phase its own finite + # budget, just as the first A child is observed before HTTP readiness. + deadline = time.monotonic() + 60 + last = "not observed" + while time.monotonic() < deadline: + try: + launch = self.fresh_status(generation, require_live=True) + last = str(launch["status"]) + if launch["status"] in {"launched", "succeeded"} and time.monotonic() < deadline: + return launch + except (subprocess.CalledProcessError, ValueError, KeyError, OSError) as error: + last = type(error).__name__ + time.sleep(0.1) + raise AssertionError("exact synthetic fresh launch was not proven; " + last) + + @staticmethod + def same_fresh_launch(first: dict[str, object], second: dict[str, object]) -> bool: + return {key: value for key, value in first.items() if key != "status"} == {key: value for key, value in second.items() if key != "status"} + + def ready(self, generation: Generation, *, fresh: bool, expected_launch: dict[str, object] | None = None) -> None: self.ensure_proxy(generation) deadline = time.monotonic() + 60 last = "not observed" @@ -550,6 +586,8 @@ def ready(self, generation: Generation, *, fresh: bool) -> None: if status["status"] not in {"launched", "succeeded"}: time.sleep(0.1) continue + if expected_launch is not None and not self.same_fresh_launch(status, expected_launch): + raise ValueError("fresh initialization launch changed after dispatch") observation = status["observation_id"] last = "unauthenticated Caddy challenge" if self.call(generation, "auto_coder_whoami", {}, authenticated=False)["http_status"] != 401: @@ -563,7 +601,7 @@ def ready(self, generation: Generation, *, fresh: bool) -> None: # sample. Require a current terminal result for that same launch. terminal = self.fresh_status(generation) last = "fresh initialization terminal status=" + str(terminal["status"]) - if {key: value for key, value in terminal.items() if key != "status"} != {key: value for key, value in status.items() if key != "status"}: + if not self.same_fresh_launch(terminal, status): raise ValueError("fresh initialization launch changed during authenticated readback") confirmed = terminal["status"] == "succeeded" if confirmed: @@ -608,7 +646,8 @@ def start_cutover(self, old: Generation, new: Generation) -> None: self.docker("start", new.container) out, err = process.communicate(timeout=60) assert process.returncode == 0, (out, err) - self.ready(new, fresh=True) + launch = self.wait_for_fresh_launch(new) + self.ready(new, fresh=True, expected_launch=launch) self.prove_child(new) def seed(self, generation: Generation): diff --git a/tests/test_management_archived_cutover_readiness.py b/tests/test_management_archived_cutover_readiness.py index 5bcacd8dd..7e20e20d0 100644 --- a/tests/test_management_archived_cutover_readiness.py +++ b/tests/test_management_archived_cutover_readiness.py @@ -89,3 +89,144 @@ def test_unauthenticated_challenge_must_still_refuse_access(monkeypatch: pytest. with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): chain.ready(generation, fresh=True) assert len(reads) == 1 and calls == [False] + + +def _cutover_case(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, ready_at: float = 80.0, replaced: str | None = None): + chain, generation, _, _ = _ready_case(monkeypatch, tmp_path) + clock = SimpleNamespace(now=0.0) + launch = {"request_id": "request-c", "owner_incarnation": "owner-c", "child_identity": "child-c", "observation_id": "a" * 32, "started": 100.0, "deadline": 220.0, "status": "launched"} + samples = [] + authenticated = [] + proved = [] + + def status(selected, *, require_live=False): + assert selected is generation + if not samples: + clock.now += 45.0 + samples.append((require_live, clock.now)) + result = {**launch, "status": "succeeded" if authenticated else "launched"} + if not require_live and (replaced == "before-auth" or (replaced == "after-auth" and authenticated)): + result["observation_id"] = "b" * 32 + return result + + def call(selected, tool, arguments, *, authenticated=True): + assert selected is generation and tool == "auto_coder_whoami" + if not authenticated: + return {"http_status": 401} + if clock.now < ready_at: + raise ValueError("application startup is still in progress") + assert arguments == {"observation_id": launch["observation_id"]} + accepted.append(clock.now) + return {"uid": "operator"} + + accepted = authenticated + monkeypatch.setattr(cutover, "time", SimpleNamespace(monotonic=lambda: clock.now, sleep=lambda _: setattr(clock, "now", clock.now + 1.0))) + monkeypatch.setattr(chain, "fresh_status", status) + monkeypatch.setattr(chain, "call", call) + monkeypatch.setattr(chain, "host", lambda selected: SimpleNamespace(communicate=lambda **kwargs: ("", ""), returncode=0)) + monkeypatch.setattr(chain, "status", lambda *args: None) + monkeypatch.setattr(chain, "stop", lambda *args: None) + monkeypatch.setattr(chain, "docker", lambda *args, **kwargs: "running") + monkeypatch.setattr(chain, "prove_child", lambda selected: proved.append(selected)) + return chain, generation, clock, samples, proved + + +def test_prelaunch_source_work_has_a_separate_finite_budget(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, clock, samples, proved = _cutover_case(monkeypatch, tmp_path) + chain.start_cutover(generation, generation) + assert samples[0] == (True, 45.0) + assert clock.now == 80.0 + assert proved == [generation] + + +@pytest.mark.parametrize("replaced", ["before-auth", "after-auth"]) +def test_launch_cannot_change_between_phase_gate_and_readiness(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, replaced: str) -> None: + chain, generation, clock, _, proved = _cutover_case(monkeypatch, tmp_path, replaced=replaced) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.start_cutover(generation, generation) + assert clock.now == 105.0 + assert proved == [] + + +def test_readiness_retries_do_not_renew_the_sixty_second_budget(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + chain, generation, clock, _, proved = _cutover_case(monkeypatch, tmp_path, ready_at=106.0) + with pytest.raises(AssertionError, match="exact synthetic child readiness was not proven"): + chain.start_cutover(generation, generation) + assert clock.now == 105.0 + assert proved == [] + + +@pytest.mark.parametrize("state", ["pending", "invalid", "unknown", "late-launch"]) +def test_launch_gate_refuses_without_renewing_its_budget(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, state: str) -> None: + chain, generation, _, _ = _ready_case(monkeypatch, tmp_path) + clock = SimpleNamespace(now=0.0) + samples = [] + + def status(selected, *, require_live=False): + assert selected is generation and require_live + samples.append(clock.now) + if state == "invalid": + raise ValueError("production fresh record or live lifetime is invalid") + if state == "late-launch": + clock.now = 61.0 + return {"status": "launched"} + return {"status": state} + + monkeypatch.setattr(cutover, "time", SimpleNamespace(monotonic=lambda: clock.now, sleep=lambda _: setattr(clock, "now", clock.now + 1.0))) + monkeypatch.setattr(chain, "fresh_status", status) + with pytest.raises(AssertionError, match="exact synthetic fresh launch was not proven"): + chain.wait_for_fresh_launch(generation) + assert 60.0 <= clock.now <= 62.0 + assert 1 <= len(samples) <= 60 + + +@pytest.mark.parametrize("mismatch", [None, "missing", "owner", "child", "phase", "expired", "window", "terminal-failed"]) +def test_live_launch_reader_joins_actual_production_observations(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, mismatch: str | None) -> None: + import io + import time + from contextlib import redirect_stdout + + from auto_coder.management import bootstrap_supervisor, fresh_initialization, self_update_handoff + + chain, generation, _, _ = _ready_case(monkeypatch, tmp_path) + child = SimpleNamespace(pid=42, start_token="start-c", child_generation=1, selected_runtime="runtime-c", supervisor_incarnation="owner-c", phase=SimpleNamespace(value="running")) + now = time.time() + launch = { + "request_id": "request-c", + "owner_incarnation": "owner-c", + "child_identity": bootstrap_supervisor.ManagementBootstrapSupervisor._identity(child), + "observation_id": "a" * 32, + "started": now - 5, + "deadline": now + 115, + "status": "launched", + } + owner = "different-owner" if mismatch == "owner" else "owner-c" + if mismatch == "child": + child.pid = 43 + if mismatch == "phase": + child.phase.value = "stopped" + if mismatch == "expired": + launch.update(started=now - 121, deadline=now - 1) + if mismatch == "window": + launch["deadline"] = now + 500 + if mismatch == "terminal-failed": + launch["status"] = "unknown" + monkeypatch.setattr(fresh_initialization, "read_fresh_status", lambda path: launch) + monkeypatch.setattr(bootstrap_supervisor.BootstrapConfig, "read", classmethod(lambda cls, path: SimpleNamespace(verification_timeout=120))) + monkeypatch.setattr(self_update_handoff, "observe_supervisor", lambda config: None if mismatch == "missing" else (owner, child)) + + def docker(*arguments, **kwargs): + assert arguments[:4] == ("exec", generation.container, "python", "-c") + captured = io.StringIO() + # Execute the actual fixture reader program with controlled production + # observations; these component tests do not claim kernel evidence. + with redirect_stdout(captured): + exec(arguments[4], {}) + return captured.getvalue() + + monkeypatch.setattr(chain, "docker", docker) + if mismatch is None: + assert chain.fresh_status(generation, require_live=True) == launch + else: + with pytest.raises(ValueError, match="fresh launch"): + chain.fresh_status(generation, require_live=True) From fb1fc14a15713945f8b0815901d12a4f2424a0d6 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 05:58:37 +0900 Subject: [PATCH 22/24] fix(test): inhibit scheduled cleanup before native recovery --- .../management-recovery-epoch.md | 9 + scripts/recovery_vm/README.md | 14 ++ scripts/recovery_vm/guest.sh | 66 ++++++ .../test_management_recovery_epoch_host.py | 2 +- tests/test_management_recovery_epoch_host.py | 10 +- tests/test_management_recovery_vm.py | 205 ++++++++++++++++++ 6 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 tests/test_management_recovery_vm.py diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md index ad11714fa..f5a27cd56 100644 --- a/docs/client-features/management-recovery-epoch.md +++ b/docs/client-features/management-recovery-epoch.md @@ -231,6 +231,15 @@ succeed. A missing case, failed VM, unavailable native kernel operation or faile preservation check cannot become a passed result. The VM runner and reviewed reproduction procedure are documented in `scripts/recovery_vm/README.md`. +The fresh recovery VM stops and masks only the measured +`systemd-tmpfiles-clean.timer` / `systemd-tmpfiles-clean.service` maintenance pair +before original producers and route approval. It records the actual stopped and +inhibited systemd facts and protected mask identities in +`maintenance-profile.json`; unavailable or changing state fails preparation. +This keeps scheduled maintenance from changing its approved execution readback +during the long native grant scenario. Production catalog/digest equality and +all ordinary or quarantined containment profiles remain unchanged. + Before recovery preparation, the fixture proves that all original A/B/C managers, proxies and cutover readers have stopped. It may then close only the empty WAL and derived SHM residue of their twelve known original stores, preserving every diff --git a/scripts/recovery_vm/README.md b/scripts/recovery_vm/README.md index aaaee3675..c048e9dba 100644 --- a/scripts/recovery_vm/README.md +++ b/scripts/recovery_vm/README.md @@ -51,6 +51,20 @@ recovery approval exists. It does not change an operational host, configure the production issuer, or establish acceptance for every stock Docker service profile or an incident with missing originals. +Before original producers or recovery approval, the guest stops and masks the +exact `systemd-tmpfiles-clean.timer` / `systemd-tmpfiles-clean.service` pair. Its +ordinary scheduled execution changed the approved service's execution readback +during the longer grant scenario. Preparation now stops the timer first, waits +for the service to stop, and uses the production systemd reader to require real +protected `/dev/null` masks, inactive state, zero service PID and no pending +daemon reload. It reobserves the inhibited facts and retains the matching result, +initial/stopped facts and mask identities in `maintenance-profile.json`, with +stdout and file logging. +An unavailable unit, failed stop, preexisting mask path or changed final state +fails preparation. This finite fresh-VM setup leaves the production whole-route +catalog and digest comparisons unchanged; it never refreshes an approval after +maintenance or changes an operational recovery host. + The guest installs the exact `uv==0.13.6` release and validates its reported release number. The complete observed `--version` banner, including its bounded build descriptor, is retained in `uv-version.txt` and supplied to staging and recovery diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh index 82ebb450a..e620c37e7 100644 --- a/scripts/recovery_vm/guest.sh +++ b/scripts/recovery_vm/guest.sh @@ -149,6 +149,72 @@ print("Installed watcher source is a private, independently copied file.") PY } verify_installed_watcher +.venv/bin/python - "$output/maintenance-profile.json" <<'PY' +import json +import os +from dataclasses import asdict +from pathlib import Path +import subprocess +import sys + +from auto_coder.logger_config import get_logger, setup_logger +from auto_coder.management.containment_route_manifest import _systemd_unit + +MAINTENANCE_UNITS = ("systemd-tmpfiles-clean.timer", "systemd-tmpfiles-clean.service") +MASK_ROOT = Path("/etc/systemd/system") +logger = get_logger(__name__) + + +def inhibit_maintenance(output): + """Settle only this fresh VM's measured maintenance pair before originals.""" + evidence = {"schema": "auto-coder-recovery-vm-maintenance-profile-v1", "status": "preparing"} + try: + masks = tuple(MASK_ROOT / unit for unit in MAINTENANCE_UNITS) + if any(path.exists() or path.is_symlink() for path in masks): + raise ValueError("The fresh maintenance mask paths are not empty") + initial = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) + evidence["initial"] = [asdict(fact) for fact in initial] + if any(not fact.unit_file_state or fact.active_state not in {"active", "inactive"} or fact.needs_daemon_reload for fact in initial): + raise ValueError("The original installed maintenance pair is unavailable") + # Stop the timer first, then wait for its service to stop. Never reset + # failed units or change an already approved recovery environment. + for unit in MAINTENANCE_UNITS: + subprocess.run(("systemctl", "stop", "--", unit), check=True, timeout=30) + stopped = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) + evidence["stopped"] = [asdict(fact) for fact in stopped] + if any(fact.active_state != "inactive" or fact.main_pid != 0 or fact.needs_daemon_reload for fact in stopped): + raise ValueError("The actual maintenance pair did not stop") + subprocess.run(("systemctl", "mask", "--", *MAINTENANCE_UNITS), check=True, timeout=30) + subprocess.run(("systemctl", "daemon-reload"), check=True, timeout=30) + observed = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) + evidence["observed"] = [asdict(fact) for fact in observed] + if any(not fact.inhibited or fact.unit_file_state != "masked" or fact.active_state != "inactive" or fact.main_pid != 0 or fact.needs_daemon_reload for fact in observed): + raise ValueError("The actual maintenance pair is not masked and inactive") + evidence["masks"] = [] + for path in masks: + info = path.lstat() + if not path.is_symlink() or os.readlink(path) != "/dev/null": + raise ValueError("The maintenance pair lacks its actual /dev/null masks") + evidence["masks"].append({"unit": path.name, "path": str(path), "device": info.st_dev, "inode": info.st_ino, "ctime_ns": info.st_ctime_ns, "mode": info.st_mode, "uid": info.st_uid, "target": os.readlink(path)}) + if tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) != observed: + raise ValueError("The inhibited maintenance pair changed during verification") + evidence["status"] = "inhibited" + logger.info("Verified the fixed maintenance timer/service pair is masked and inactive before original fixture creation") + except Exception as error: + evidence.update(status="failed", failure_type=type(error).__name__) + logger.error("Fresh recovery VM maintenance preparation failed: {}", type(error).__name__) + raise + finally: + with output.open("x") as stream: + json.dump(evidence, stream, sort_keys=True) + + +if __name__ == "__main__": + output = Path(sys.argv[1]) + setup_logger(stream=sys.stdout, log_file=str(output.with_suffix(".log"))) + inhibit_maintenance(output) + logger.complete() +PY docker compose version install -m 600 "$input/source-token" /run/recovery-fixture-source-token mapfile -t selected < <(python3 - "$input" <<'PY' diff --git a/scripts/test_management_recovery_epoch_host.py b/scripts/test_management_recovery_epoch_host.py index 626d60ab7..47ccaf187 100644 --- a/scripts/test_management_recovery_epoch_host.py +++ b/scripts/test_management_recovery_epoch_host.py @@ -63,7 +63,7 @@ LINEAGE_REPORT_CASES = frozenset({"forged-original-lineage-refused", "stale-original-lineage-refused", "cross-deployment-original-lineage-refused"}) FACTORY_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES if case.startswith("factory-")) TASK_OBSERVATION_CASES = frozenset(case for case in REQUIRED_NEGATIVE_CASES | REQUIRED_REPORT_CASES if "nonleader" in case) -ACCEPTANCE_SCENARIOS = ("grant", "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS, *LATE_MUTATION_SCENARIOS, *POST_RECEIPT_MUTATION_SCENARIOS) +ACCEPTANCE_SCENARIOS = ("grant", *POST_RECEIPT_MUTATION_SCENARIOS, "lost-reply", "retarget-after-reservation", *BOUNDARY_SCENARIOS, *LATE_MUTATION_SCENARIOS) SCENARIOS = (*ACCEPTANCE_SCENARIOS, "kill-after-intent", "kill-after-consume") REQUIRED_CASES = frozenset( { diff --git a/tests/test_management_recovery_epoch_host.py b/tests/test_management_recovery_epoch_host.py index 5b3be5d68..410626426 100644 --- a/tests/test_management_recovery_epoch_host.py +++ b/tests/test_management_recovery_epoch_host.py @@ -15,6 +15,7 @@ from pathlib import Path import pytest +import yaml from scripts import recovery_epoch_native_negatives as negatives from scripts import test_management_recovery_epoch_host as fixture @@ -541,8 +542,15 @@ def test_unknown_native_scenario_is_not_a_success_alias() -> None: def test_canonical_acceptance_contains_all_distinct_publication_and_late_faults() -> None: - assert len(set(fixture.ACCEPTANCE_SCENARIOS)) == len(fixture.ACCEPTANCE_SCENARIOS) + assert len(set(fixture.ACCEPTANCE_SCENARIOS)) == len(fixture.ACCEPTANCE_SCENARIOS) == 31 assert set(fixture.ACCEPTANCE_SCENARIOS) == {"grant", "lost-reply", "retarget-after-reservation", *fixture.BOUNDARY_SCENARIOS, *fixture.LATE_MUTATION_SCENARIOS, *fixture.POST_RECEIPT_MUTATION_SCENARIOS} + assert fixture.ACCEPTANCE_SCENARIOS[: 1 + len(fixture.POST_RECEIPT_MUTATION_SCENARIOS)] == ("grant", *fixture.POST_RECEIPT_MUTATION_SCENARIOS) + assert fixture.CONTROL not in fixture.ACCEPTANCE_SCENARIOS + jobs = yaml.safe_load((Path(__file__).resolve().parents[1] / ".github/workflows/management-recovery-epoch.yml").read_text())["jobs"] + assert jobs["recovery-epoch"]["strategy"]["matrix"]["scenario"] == "${{ fromJSON(needs.acceptance-plan.outputs.scenarios) }}" + assert jobs["verifier-omission-control"]["env"]["RECOVERY_SCENARIO"] in fixture.POST_RECEIPT_MUTATION_SCENARIOS + assert jobs["recovery-epoch"]["env"]["RECOVERY_FIXTURE"] != jobs["verifier-omission-control"]["env"]["RECOVERY_FIXTURE"] + assert set(jobs["acceptance"]["needs"]) == {"acceptance-plan", "recovery-epoch", "verifier-omission-control"} assert set(fixture.SCENARIOS) - set(fixture.ACCEPTANCE_SCENARIOS) == {"kill-after-intent", "kill-after-consume"} assert fixture._selected_boundary("post-receipt-ready-mutation") == ("receipt-head.json", "after") assert all(fixture._selected_boundary(name) == ("intent-head.json", "after") for name in fixture.LATE_MUTATION_SCENARIOS) diff --git a/tests/test_management_recovery_vm.py b/tests/test_management_recovery_vm.py new file mode 100644 index 000000000..654569f39 --- /dev/null +++ b/tests/test_management_recovery_vm.py @@ -0,0 +1,205 @@ +"""Component checks execute the real guest setup and systemd fact parser.""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from auto_coder.management import containment_route_manifest as routes + +GUEST = Path(__file__).resolve().parents[1] / "scripts/recovery_vm/guest.sh" +MAINTENANCE_COMMAND = ".venv/bin/python - \"$output/maintenance-profile.json\" <<'PY'\n" +TIMER = "systemd-tmpfiles-clean.timer" +SERVICE = "systemd-tmpfiles-clean.service" +UNITS = (TIMER, SERVICE) + + +class SystemdFixture: + """Supply systemctl readback while retaining real parser and mask objects.""" + + def __init__(self, root: Path, fault: str | None = None) -> None: + self.root = root + self.fault = fault + self.calls: list[tuple[str, ...]] = [] + self.stopped: set[str] = set() + self.masked = False + self.reads = {unit: 0 for unit in UNITS} + self.timer_rows: list[dict[str, str]] = [] + + def command(self, *arguments: str) -> str: + assert arguments[0] == "systemctl" and arguments[-1] in UNITS + unit = arguments[-1] + if arguments[1] == "cat": + return "# Installed maintenance unit\n[Service]\nExecStart=/usr/bin/systemd-tmpfiles --clean\n" + assert arguments[1] == "show" + state = { + "Id": unit, + "LoadState": "masked" if self.masked else "loaded", + "UnitFileState": "masked" if self.masked else "static", + "ActiveState": "inactive" if unit in self.stopped else "active", + "SubState": "dead" if unit in self.stopped else "running" if unit == SERVICE else "waiting", + "NeedDaemonReload": "no", + "FragmentPath": str(self.root / unit) if self.masked else "/usr/lib/systemd/system/" + unit, + } + if unit == SERVICE: + state["MainPID"] = "0" if unit in self.stopped else "123" + if not self.masked and self.fault == "missing" and unit == SERVICE: + state.update(LoadState="not-found", UnitFileState="", ActiveState="inactive", FragmentPath="") + if not self.masked and unit in self.stopped and self.fault == "still-running" and unit == SERVICE: + state.update(ActiveState="active", SubState="running", MainPID="123") + if self.masked: + self.reads[unit] += 1 + if unit == SERVICE and self.fault and self.fault.startswith("property:"): + _, key, value = self.fault.split(":", 2) + if value == "missing": + state.pop(key) + else: + state[key] = value + if unit == TIMER and self.fault == "changed-mask" and self.reads[unit] == 2: + path = self.root / unit + path.rename(self.root / "retained-original-mask") + path.symlink_to("/dev/null") + if unit == TIMER: + self.timer_rows.append(state.copy()) + return "\n".join(key + "=" + value for key, value in state.items()) + + def run(self, command: tuple[str, ...], *, check: bool, timeout: int) -> subprocess.CompletedProcess[str]: + assert check is True and timeout == 30 + self.calls.append(command) + if command[:3] == ("systemctl", "stop", "--"): + unit = command[3] + assert unit in UNITS + if self.fault == "stop-failed" and unit == SERVICE: + raise subprocess.CalledProcessError(1, command) + self.stopped.add(unit) + elif command == ("systemctl", "mask", "--", *UNITS): + assert self.stopped == set(UNITS) + for unit in UNITS: + path = self.root / unit + if self.fault == "regular-mask" and unit == SERVICE: + path.write_text("not a mask") + else: + path.symlink_to("/dev/zero" if self.fault == "wrong-mask" and unit == SERVICE else "/dev/null") + self.masked = True + else: + assert command == ("systemctl", "daemon-reload") and self.masked + return subprocess.CompletedProcess(command, 0, "", "") + + +@pytest.fixture +def maintenance(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + text = GUEST.read_text() + assert text.count(MAINTENANCE_COMMAND) == 1 + body = text.split(MAINTENANCE_COMMAND, 1)[1].split("\nPY\n", 1)[0] + namespace = {"__name__": "recovery_vm_maintenance_component"} + exec(compile(body, str(GUEST), "exec"), namespace) + mask_root = tmp_path / "masks" + mask_root.mkdir() + namespace["MASK_ROOT"] = mask_root + actual_reader = routes._systemd_unit + namespace["_systemd_unit"] = lambda unit: actual_reader(unit, trusted_uid=os.geteuid()) + + def prepare(fault: str | None = None) -> SystemdFixture: + fixture = SystemdFixture(mask_root, fault) + monkeypatch.setattr(routes, "_command", fixture.command) + namespace["subprocess"] = SimpleNamespace(run=fixture.run) + return fixture + + return namespace["inhibit_maintenance"], prepare, tmp_path / "maintenance-profile.json", mask_root + + +def test_guest_inhibits_only_exact_pair_before_originals_and_control_copy() -> None: + text = GUEST.read_text() + setup = text.index(MAINTENANCE_COMMAND) + assert text.index("verify_installed_watcher\n") < setup + assert setup < text.index("driver_module=scripts.test_management_recovery_epoch_host") + assert setup < text.index("scripts.recovery_epoch_mutation_control prepare") + assert setup < text.index('fd = os.open("/etc/auto-coder-disposable-containment-host.json"') + assert setup < text.index('.venv/bin/python -m "$driver_module"') + subprocess.run(("bash", "-n", str(GUEST)), check=True, timeout=10) + + +def test_real_guest_block_stops_timer_first_and_retains_actual_masks(maintenance) -> None: + inhibit, prepare, output, masks = maintenance + systemd = prepare() + inhibit(output) + assert systemd.calls == [ + ("systemctl", "stop", "--", TIMER), + ("systemctl", "stop", "--", SERVICE), + ("systemctl", "mask", "--", *UNITS), + ("systemctl", "daemon-reload"), + ] + evidence = json.loads(output.read_text()) + assert evidence["schema"] == "auto-coder-recovery-vm-maintenance-profile-v1" and evidence["status"] == "inhibited" + assert [fact["unit"] for fact in evidence["observed"]] == list(UNITS) + assert evidence["initial"][1]["main_pid"] == 123 + assert all(fact["active_state"] == "inactive" and fact["main_pid"] == 0 for fact in evidence["stopped"]) + assert all(fact["inhibited"] and fact["unit_file_state"] == "masked" and fact["main_pid"] == 0 and not fact["needs_daemon_reload"] for fact in evidence["observed"]) + assert all("MainPID" not in row for row in systemd.timer_rows) + assert systemd.reads == {TIMER: 2, SERVICE: 2} + for record in evidence["masks"]: + path = masks / record["unit"] + info = path.lstat() + assert record == {"unit": path.name, "path": str(path), "device": info.st_dev, "inode": info.st_ino, "ctime_ns": info.st_ctime_ns, "mode": info.st_mode, "uid": info.st_uid, "target": "/dev/null"} + assert os.readlink(path) == "/dev/null" + + +def test_existing_mask_is_preserved_without_stopping_or_resetting(maintenance) -> None: + inhibit, prepare, output, masks = maintenance + path = masks / TIMER + path.symlink_to("/dev/null") + before = path.lstat() + systemd = prepare() + with pytest.raises(ValueError, match="mask paths are not empty"): + inhibit(output) + assert path.lstat() == before and os.readlink(path) == "/dev/null" + assert systemd.calls == [] + assert json.loads(output.read_text())["status"] == "failed" + + +@pytest.mark.parametrize("fault", ("missing", "stop-failed", "still-running")) +def test_missing_or_unstopped_service_never_installs_a_mask(maintenance, fault: str) -> None: + inhibit, prepare, output, masks = maintenance + systemd = prepare(fault) + with pytest.raises((ValueError, subprocess.CalledProcessError)): + inhibit(output) + assert not tuple(masks.iterdir()) + assert all(command[1] == "stop" for command in systemd.calls) + if fault == "missing": + assert systemd.calls == [] + evidence = json.loads(output.read_text()) + assert evidence["status"] == "failed" and evidence["failure_type"] + assert "observed" not in evidence + + +@pytest.mark.parametrize( + "fault", + ( + "property:LoadState:loaded", + "property:UnitFileState:static", + "property:ActiveState:active", + "property:SubState:running", + "property:MainPID:123", + "property:MainPID:missing", + "property:NeedDaemonReload:yes", + "regular-mask", + "wrong-mask", + "changed-mask", + ), +) +def test_guest_refuses_incomplete_or_changed_actual_inhibition(maintenance, fault: str) -> None: + inhibit, prepare, output, masks = maintenance + systemd = prepare(fault) + with pytest.raises(ValueError): + inhibit(output) + evidence = json.loads(output.read_text()) + assert evidence["status"] == "failed" and evidence["failure_type"] + assert systemd.calls[-1] == ("systemctl", "daemon-reload") + assert len(systemd.calls) == 4 + assert (masks / TIMER).is_symlink() + assert all("reset-failed" not in command and "unmask" not in command for command in systemd.calls) From d9d8e9d004acc12964bb648da015e74020254f39 Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 07:33:37 +0900 Subject: [PATCH 23/24] fix: preserve evidence across management observation races Defer a Supervisor change seen at the second lifecycle check to the existing retained-receipt gate without a terminal write or redispatch. For independently proven proc inventory churn, discard incomplete cron observations and require a fresh matching pair within three attempts. Retain protected source, controller, boot and mask refusal anchors and every hard task/evidence refusal. --- ...agement-current-containment-observation.md | 11 + .../management/containment_cron_inhibition.py | 161 +++++++++---- src/auto_coder/management/worker_lifecycle.py | 4 +- ..._management_containment_cron_inhibition.py | 213 ++++++++++++++++++ tests/test_management_worker_lifecycle.py | 92 ++++++++ 5 files changed, 439 insertions(+), 42 deletions(-) diff --git a/docs/client-features/management-current-containment-observation.md b/docs/client-features/management-current-containment-observation.md index 0b30b65c9..80722d32e 100644 --- a/docs/client-features/management-current-containment-observation.md +++ b/docs/client-features/management-current-containment-observation.md @@ -275,6 +275,17 @@ fresh reads. A process running a scheduler outside its normal service also refuses. This initial profile conservatively refuses unrelated masked services that retain work as well. +A process or thread may disappear while the complete procfs inventory is read. +Only disappearance independently confirmed by a fresh PID/TID inventory, or a +changed complete inventory, permits a retry. At most three attempts are made; +each discards the incomplete pair and reads two new complete matching +source/controller/cgroup/task snapshots. Earlier protected source, controller, +boot and mask bindings remain refusal anchors across attempts. A still-listed +process or thread with missing evidence, an empty live task inventory, permission +or malformed-data errors, changed protected facts, and any positively observed +inhibited task or scheduler still refuse immediately. No task is skipped to +complete a positive observation, and persistent inventory churn remains unavailable. + A detected remaining task has a typed refusal at the route boundary. Its public diagnostic contains fixed text and a bounded validated controller-scope identifier so an independent consumer can distinguish the canonical and differently named diff --git a/src/auto_coder/management/containment_cron_inhibition.py b/src/auto_coder/management/containment_cron_inhibition.py index 931139472..414efdc64 100644 --- a/src/auto_coder/management/containment_cron_inhibition.py +++ b/src/auto_coder/management/containment_cron_inhibition.py @@ -62,6 +62,7 @@ _MAX_GROUPS = 8192 _MAX_TASKS = 131072 _MAX_SERVICES = 4096 +_MAX_OBSERVATION_ATTEMPTS = 3 # systemd v255 cg_needs_escape also prefixes controller.* cgroup names. _ESCAPED_SERVICE_PREFIXES = ("cgroup", "cpu", "cpuacct", "cpuset", "io", "blkio", "memory", "devices", "pids", "bpf-firewall", "bpf-devices", "bpf-foreign", "bpf-socket-bind", "bpf-restrict-network-interfaces") _PROPERTIES = ("Id", "LoadState", "UnitFileState", "ActiveState", "SubState", "MainPID", "ControlPID", "NeedDaemonReload", "FragmentPath", "SourcePath", "DropInPaths", "ControlGroup", "Slice", "Delegate") @@ -71,6 +72,14 @@ class CronInhibitionUnavailable(ValueError): """The bounded stock cron execution scope is not currently closed.""" +class _ProcInventoryChanged(CronInhibitionUnavailable): + """An independently changed inventory invalidated the entire observation.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.source_binding: _CronSourceBinding | None = None + + class CronScopeRetainsTasks(CronInhibitionUnavailable): """A measured task predicate failed, with a separate safe scope field.""" @@ -223,6 +232,19 @@ def parse(cls, value: object) -> CronInhibitedSource: return cls(PROFILE, value["boot_id"], "cron.service", value["scheduler_sha256"], properties, sources, value["mask_path"], tuple(mask), CronCgroupScope.parse(value["cgroups"], inhibited_units=units), services) +@dataclass(frozen=True) +class _CronSourceBinding: + boot_id: str + controller_properties: tuple[tuple[str, str], ...] + sources: tuple[CommandSourceFact, ...] + mask_path: str + mask_identity: tuple[int, ...] + + @classmethod + def from_observation(cls, value: CronInhibitedSource) -> _CronSourceBinding: + return cls(value.boot_id, value.controller_properties, value.sources, value.mask_path, value.mask_identity) + + def _validate_service_units(units: tuple[str, ...]) -> None: # Literal service names map to literal cgroup components. Templates, # instances, escaping and reserved controller prefixes need other profiles. @@ -325,44 +347,70 @@ def _empty_group(path: Path, relative: str, *, scope: str | None = None) -> Empt return EmptyCgroupFact(relative, (), (), 0, int(values["frozen"])) +def _proc_inventory(path: Path) -> tuple[str, ...]: + return tuple(sorted(entry.name for entry in path.iterdir() if entry.name.isdecimal())) + + +def _proc_identity_disappeared(proc: Path, pid: str, tid: str | None) -> bool: + if pid not in _proc_inventory(proc): + return True + if tid is None: + return False + try: + return tid not in _proc_inventory(proc / pid / "task") + except FileNotFoundError: + # A live process with an unavailable task directory is not an exit. + return pid not in _proc_inventory(proc) + + def _proc_scopes_empty(roots: tuple[str, ...], *, proc: Path = Path("/proc"), inhibited_units: tuple[str, ...] = ("cron.service",)) -> None: - pids = tuple(sorted(path.name for path in proc.iterdir() if path.name.isdecimal())) + pids = _proc_inventory(proc) count = 0 for pid in pids: task_root = proc / pid / "task" - tids = tuple(sorted(path.name for path in task_root.iterdir() if path.name.isdecimal())) - for tid in tids: - count += 1 - if count > _MAX_TASKS: - raise CronInhibitionUnavailable("stock cron task inventory exceeds its bound") - task = task_root / tid - before = _read(task / "stat", 16384) - tail = before.rpartition(")")[2].split() - if len(tail) < 20 or not tail[19].isdecimal(): - raise CronInhibitionUnavailable("stock cron task lifetime is unavailable") - groups = _read(task / "cgroup", 16384).splitlines() - if len(groups) != 1 or not groups[0].startswith("0::/"): - raise CronInhibitionUnavailable("stock cron task cgroup membership is unsupported") - group = groups[0][3:] - if any(group == root or group.startswith(root + "/") for root in roots) or any(part in inhibited_units for part in Path(group).parts): - raise CronScopeRetainsTasks("inhibited stock cron scope retains tasks: pid=" + pid + " tid=" + tid + " state=" + tail[0] + " start=" + tail[19], scope=_retained_task_scope(group, roots, inhibited_units)) - after = _read(task / "stat", 16384).rpartition(")")[2].split() - if len(after) < 20 or after[19] != tail[19] or _read(task / "cgroup", 16384).splitlines() != groups: - raise CronInhibitionUnavailable("stock cron task changed during scope inspection") - if tids != tuple(sorted(path.name for path in task_root.iterdir() if path.name.isdecimal())): - raise CronInhibitionUnavailable("stock cron thread inventory changed during inspection") - # A daemon started outside its normal unit still remains an ordinary - # scheduler, even if that service's own cgroup is empty. - with (proc / pid / "cmdline").open("rb") as stream: - command = stream.read(131073) - if len(command) > 131072: - raise CronInhibitionUnavailable("stock cron process command exceeds its bound") - if command: - executable = os.readlink(proc / pid / "exe") - if executable.removesuffix(" (deleted)") in ("/usr/sbin/cron", *_OTHER_EXECUTABLES): - raise CronScopeRetainsTasks("inhibited stock cron scope retains tasks: scheduler pid=" + pid) - if pids != tuple(sorted(path.name for path in proc.iterdir() if path.name.isdecimal())): - raise CronInhibitionUnavailable("stock cron process inventory changed during inspection") + active_tid = None + try: + tids = _proc_inventory(task_root) + if not tids or pid not in tids: + raise CronInhibitionUnavailable("stock cron live process task inventory is incomplete") + for tid in tids: + active_tid = tid + count += 1 + if count > _MAX_TASKS: + raise CronInhibitionUnavailable("stock cron task inventory exceeds its bound") + task = task_root / tid + before = _read(task / "stat", 16384) + tail = before.rpartition(")")[2].split() + if len(tail) < 20 or not tail[19].isdecimal(): + raise CronInhibitionUnavailable("stock cron task lifetime is unavailable") + groups = _read(task / "cgroup", 16384).splitlines() + if len(groups) != 1 or not groups[0].startswith("0::/"): + raise CronInhibitionUnavailable("stock cron task cgroup membership is unsupported") + group = groups[0][3:] + if any(group == root or group.startswith(root + "/") for root in roots) or any(part in inhibited_units for part in Path(group).parts): + raise CronScopeRetainsTasks("inhibited stock cron scope retains tasks: pid=" + pid + " tid=" + tid + " state=" + tail[0] + " start=" + tail[19], scope=_retained_task_scope(group, roots, inhibited_units)) + after = _read(task / "stat", 16384).rpartition(")")[2].split() + if len(after) < 20 or after[19] != tail[19] or _read(task / "cgroup", 16384).splitlines() != groups: + raise CronInhibitionUnavailable("stock cron task changed during scope inspection") + active_tid = None + if tids != _proc_inventory(task_root): + raise _ProcInventoryChanged("stock cron thread inventory changed during inspection") + # A daemon started outside its normal unit remains an ordinary + # scheduler, even if that service's own cgroup is empty. + with (proc / pid / "cmdline").open("rb") as stream: + command = stream.read(131073) + if len(command) > 131072: + raise CronInhibitionUnavailable("stock cron process command exceeds its bound") + if command: + executable = os.readlink(proc / pid / "exe") + if executable.removesuffix(" (deleted)") in ("/usr/sbin/cron", *_OTHER_EXECUTABLES): + raise CronScopeRetainsTasks("inhibited stock cron scope retains tasks: scheduler pid=" + pid) + except FileNotFoundError as exc: + if _proc_identity_disappeared(proc, pid, active_tid): + raise _ProcInventoryChanged("stock cron process/thread inventory changed during inspection") from exc + raise + if pids != _proc_inventory(proc): + raise _ProcInventoryChanged("stock cron process inventory changed during inspection") def _cgroup_snapshot(*, root: Path = _CGROUP_ROOT, mountinfo: Path = Path("/proc/self/mountinfo"), proc: Path = Path("/proc"), inhibited_units: tuple[str, ...] = ("cron.service",)) -> CronCgroupScope: @@ -455,7 +503,7 @@ def _properties() -> tuple[tuple[str, str], ...]: return tuple(sorted(values.items())) -def _snapshot(scheduler_sha256: str, inhibited_services: tuple[tuple[str, str], ...]) -> CronInhibitedSource: +def _source_snapshot() -> _CronSourceBinding: boot = _read(Path("/proc/sys/kernel/random/boot_id"), 128).strip() if not re.fullmatch(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", boot): raise CronInhibitionUnavailable("stock cron boot identity is unavailable") @@ -466,7 +514,17 @@ def _snapshot(scheduler_sha256: str, inhibited_services: tuple[tuple[str, str], if not stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or os.readlink(mask) != "/dev/null": raise CronInhibitionUnavailable("stock cron protected mask is unavailable") identity = (info.st_dev, info.st_ino, info.st_ctime_ns, info.st_mode, info.st_uid, info.st_gid) - return CronInhibitedSource(PROFILE, boot, "cron.service", scheduler_sha256, properties, _profile_sources(), str(mask), identity, _cgroup_snapshot(inhibited_units=tuple(unit for unit, _ in inhibited_services)), inhibited_services) + return _CronSourceBinding(boot, properties, _profile_sources(), str(mask), identity) + + +def _snapshot(scheduler_sha256: str, inhibited_services: tuple[tuple[str, str], ...]) -> CronInhibitedSource: + source = _source_snapshot() + try: + cgroups = _cgroup_snapshot(inhibited_units=tuple(unit for unit, _ in inhibited_services)) + except _ProcInventoryChanged as exc: + exc.source_binding = source + raise + return CronInhibitedSource(PROFILE, source.boot_id, "cron.service", scheduler_sha256, source.controller_properties, source.sources, source.mask_path, source.mask_identity, cgroups, inhibited_services) def observe_inhibited_stock_cron_scope(systemd_facts: tuple[object, ...]) -> CronInhibitedSource: @@ -485,10 +543,33 @@ def observe_inhibited_stock_cron_scope(systemd_facts: tuple[object, ...]) -> Cro raise CronInhibitionUnavailable("stock cron controller observation identity is unavailable") try: services = _inhibited_service_bindings(systemd_facts) - first = _snapshot(digest, services) - if _snapshot(digest, services) != first: - raise CronInhibitionUnavailable("stock cron source/controller/task scope changed during observation") - return first + source_binding: _CronSourceBinding | None = None + + def retain_source_binding(current: _CronSourceBinding) -> None: + nonlocal source_binding + if source_binding is not None and current != source_binding: + raise CronInhibitionUnavailable("stock cron source/controller authority changed during observation") + source_binding = current + + for attempt in range(_MAX_OBSERVATION_ATTEMPTS): + try: + first = _snapshot(digest, services) + retain_source_binding(_CronSourceBinding.from_observation(first)) + second = _snapshot(digest, services) + retain_source_binding(_CronSourceBinding.from_observation(second)) + except _ProcInventoryChanged as exc: + if exc.source_binding is None: + raise CronInhibitionUnavailable("stock cron incomplete observation lacks its source binding") from exc + retain_source_binding(exc.source_binding) + if attempt == _MAX_OBSERVATION_ATTEMPTS - 1: + raise CronInhibitionUnavailable("stock cron process/thread inventory did not stabilize") from exc + # No earlier successful or partial task snapshot supplies any + # part of the next positive observation pair. + continue + if second != first: + raise CronInhibitionUnavailable("stock cron source/controller/task scope changed during observation") + return first + raise CronInhibitionUnavailable("stock cron complete observation pair is unavailable") except (OSError, ValueError, subprocess.SubprocessError) as exc: if isinstance(exc, CronInhibitionUnavailable): raise diff --git a/src/auto_coder/management/worker_lifecycle.py b/src/auto_coder/management/worker_lifecycle.py index e15f3be1a..6a3ca1f83 100644 --- a/src/auto_coder/management/worker_lifecycle.py +++ b/src/auto_coder/management/worker_lifecycle.py @@ -713,9 +713,9 @@ def _step(self, record: OperationRecord, version: int, snapshot: dict[str, objec steps = snapshot["steps"] assert isinstance(steps, dict) identity = self._original_supervisor(client, snapshot) - if identity == "CHANGED": - return self._finish(record, version, snapshot, "NEEDS_ATTENTION", "original_supervisor_incarnation_changed") if identity != "SAME": + # Reconstruction can occur after _process_locked's first check. + # Its next iteration must apply the original retained-receipt gate. return record if name not in steps: if not self.runtime.retained_effect_permitted(record): diff --git a/tests/test_management_containment_cron_inhibition.py b/tests/test_management_containment_cron_inhibition.py index a66511f35..183d8e825 100644 --- a/tests/test_management_containment_cron_inhibition.py +++ b/tests/test_management_containment_cron_inhibition.py @@ -10,6 +10,7 @@ import json import os import pwd +import shutil import stat import tempfile from contextlib import contextmanager @@ -336,6 +337,218 @@ def test_profile_reobserves_complete_scope_and_rejects_races(closed_proof, monke cron.observe_inhibited_stock_cron_scope((unit,)) +def _write_proc_task(proc: Path, *, pid: str = "1089", tid: str = "1089", group: str = "/system.slice/unrelated.service") -> Path: + task = proc / pid / "task" / tid + task.mkdir(parents=True) + (task / "stat").write_text(tid + " (fixture) S " + "0 " * 18 + "777 0\n") + (task / "cgroup").write_text("0::" + group + "\n") + if not (proc / pid / "cmdline").exists(): + (proc / pid / "cmdline").write_bytes(b"fixture\0") + if not (proc / pid / "exe").is_symlink(): + (proc / pid / "exe").symlink_to("/usr/bin/sleep") + return task + + +def _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch): + """Keep the real paired observer and filesystem scope reader in this component test.""" + root, proc, mountinfo = scope_tree + state = SimpleNamespace(source=cron._CronSourceBinding.from_observation(closed_proof), source_values=[], source_reads=0, scope_reads=0, completed=[], before_scope=lambda: None) + read_scope = cron._cgroup_snapshot + + def source(): + state.source_reads += 1 + return state.source_values.pop(0) if state.source_values else state.source + + def scope(*, inhibited_units): + state.scope_reads += 1 + state.before_scope() + value = read_scope(root=root, proc=proc, mountinfo=mountinfo, inhibited_units=inhibited_units) + state.completed.append(value) + return value + + monkeypatch.setattr(cron, "_source_snapshot", source) + monkeypatch.setattr(cron, "_cgroup_snapshot", scope) + state.facts = (routes.SystemdRouteFact("cron.service", "a" * 64, "masked", "inactive", 0, True),) + return state + + +@pytest.mark.parametrize("incomplete_snapshot", (1, 2)) +def test_disappearing_process_discards_the_entire_observation_pair(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, incomplete_snapshot: int) -> None: + root, proc, _ = scope_tree + _write_proc_task(proc) + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + enumerate_directory = Path.iterdir + injected = False + + def disappear(path): + nonlocal injected + if path == proc / "1089/task" and state.scope_reads == incomplete_snapshot and not injected: + injected = True + shutil.rmtree(proc / "1089") + _write_group(root / "system.slice/after-exit.scope") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", disappear) + observed = cron.observe_inhibited_stock_cron_scope(state.facts) + assert injected and state.source_reads == state.scope_reads == incomplete_snapshot + 2 + assert state.completed[-2] == state.completed[-1] == observed.cgroups + assert "/system.slice/after-exit.scope" in {item.path for item in observed.cgroups.directories} + if incomplete_snapshot == 2: + assert state.completed[0] != observed.cgroups + + +def test_disappearing_nonleader_is_reobserved_without_skipping_other_tasks(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc) + thread = _write_proc_task(proc, tid="1090") + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + read = cron._read + injected = False + + def disappear(path, limit=1024 * 1024): + nonlocal injected + if path == thread / "stat" and not injected: + injected = True + shutil.rmtree(thread) + return read(path, limit) + + monkeypatch.setattr(cron, "_read", disappear) + observed = cron.observe_inhibited_stock_cron_scope(state.facts) + assert injected and state.source_reads == state.scope_reads == 3 + assert observed.cgroups == state.completed[-2] == state.completed[-1] + assert (proc / "1089/task/1089/stat").is_file() + + +def test_repeated_process_churn_has_a_finite_complete_observation_budget(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + _, proc, _ = scope_tree + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + state.before_scope = lambda: _write_proc_task(proc) + enumerate_directory = Path.iterdir + + def disappear(path): + if path == proc / "1089/task": + shutil.rmtree(proc / "1089") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", disappear) + with pytest.raises(cron.CronInhibitionUnavailable, match="inventory did not stabilize"): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 3 + assert not state.completed + + +@pytest.mark.parametrize("fault", ("task-directory", "empty-tasks", "stat", "cgroup", "nonleader-stat", "nonleader-cgroup", "cmdline", "exe", "malformed-stat", "malformed-cgroup", "permission", "eperm")) +def test_live_missing_or_invalid_process_evidence_is_not_retried(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, fault: str) -> None: + _, proc, _ = scope_tree + task = _write_proc_task(proc) + if fault.startswith("nonleader-"): + task = _write_proc_task(proc, tid="1090") + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + if fault == "task-directory": + shutil.rmtree(task.parent) + elif fault == "empty-tasks": + shutil.rmtree(task) + elif fault in {"stat", "cgroup", "nonleader-stat", "nonleader-cgroup"}: + (task / fault.removeprefix("nonleader-")).unlink() + elif fault in {"cmdline", "exe"}: + (proc / "1089" / fault).unlink() + elif fault.startswith("malformed-"): + (task / fault.removeprefix("malformed-")).write_text("unavailable\n") + else: + read = cron._read + + def denied(path, limit=1024 * 1024): + if path == task / "stat": + raise PermissionError(1 if fault == "eperm" else 13, "fixture task is inaccessible") + return read(path, limit) + + monkeypatch.setattr(cron, "_read", denied) + with pytest.raises(cron.CronInhibitionUnavailable): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 1 + + +@pytest.mark.parametrize("kind", ("inhibited-membership", "scheduler")) +def test_positively_observed_inhibited_tasks_are_never_retried(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, kind: str) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc, group="/system.slice/cron.service" if kind == "inhibited-membership" else "/system.slice/unrelated.service") + if kind == "scheduler": + (proc / "1089/exe").unlink() + (proc / "1089/exe").symlink_to("/usr/sbin/cron") + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + with pytest.raises(cron.CronScopeRetainsTasks): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 1 + + +def test_new_nonleader_in_an_inhibited_scope_is_found_by_the_fresh_complete_scan(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc) + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + enumerate_directory = Path.iterdir + task_reads = 0 + + def add_thread(path): + nonlocal task_reads + if path == proc / "1089/task": + task_reads += 1 + if task_reads == 2: + _write_proc_task(proc, tid="1090", group="/system.slice/cron.service") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", add_thread) + with pytest.raises(cron.CronScopeRetainsTasks, match="tid=1090"): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 2 + assert not state.completed + + +@pytest.mark.parametrize("field", ("boot_id", "controller_properties", "sources", "mask_path", "mask_identity")) +def test_changed_source_binding_cannot_be_hidden_by_a_second_snapshot_disappearance(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch, field: str) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc) + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + changes = { + "boot_id": "11111111-89ab-cdef-0123-456789abcdef", + "controller_properties": tuple((key, "changed" if key == "FragmentPath" else value) for key, value in state.source.controller_properties), + "sources": (replace(state.source.sources[0], inode=99), *state.source.sources[1:]), + "mask_path": "/run/systemd/system/cron.service", + "mask_identity": (1, 99, 3, stat.S_IFLNK | 0o777, 0, 0), + } + state.source_values = [state.source, replace(state.source, **{field: changes[field]})] + enumerate_directory = Path.iterdir + + def disappear(path): + if path == proc / "1089/task" and state.scope_reads == 2: + shutil.rmtree(proc / "1089") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", disappear) + with pytest.raises(cron.CronInhibitionUnavailable, match="source/controller authority changed"): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 2 + assert len(state.completed) == 1 + + +def test_first_incomplete_snapshot_retains_its_protected_source_refusal_anchor(scope_tree, closed_proof, monkeypatch: pytest.MonkeyPatch) -> None: + _, proc, _ = scope_tree + _write_proc_task(proc) + state = _filesystem_cron_observer(scope_tree, closed_proof, monkeypatch) + state.source_values = [state.source, replace(state.source, mask_identity=(1, 99, 3, stat.S_IFLNK | 0o777, 0, 0))] + enumerate_directory = Path.iterdir + + def disappear(path): + if path == proc / "1089/task" and state.scope_reads == 1: + shutil.rmtree(proc / "1089") + return enumerate_directory(path) + + monkeypatch.setattr(Path, "iterdir", disappear) + with pytest.raises(cron.CronInhibitionUnavailable, match="source/controller authority changed"): + cron.observe_inhibited_stock_cron_scope(state.facts) + assert state.source_reads == state.scope_reads == 2 + assert len(state.completed) == 1 + + def test_vendor_source_read_binds_actual_protected_file_and_refuses_change(native_sources: dict[str, str]) -> None: with tempfile.TemporaryDirectory(prefix="ac-cron-vendor-", dir=pwd.getpwuid(os.geteuid()).pw_dir) as directory: path = Path(directory) / "cron.service" diff --git a/tests/test_management_worker_lifecycle.py b/tests/test_management_worker_lifecycle.py index fcbf16e59..f420889c8 100644 --- a/tests/test_management_worker_lifecycle.py +++ b/tests/test_management_worker_lifecycle.py @@ -8,6 +8,7 @@ import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from types import SimpleNamespace import pytest from starlette.testclient import TestClient @@ -730,6 +731,97 @@ def observe_status(): assert status_entered.is_set() +@pytest.mark.parametrize("receipt_state", ("complete", "pending", "digest-conflict", "missing", "expired-pending")) +def test_supervisor_reconstruction_between_identity_checks_retains_original_effect(real_boundary, tmp_path: Path, monkeypatch, receipt_state: str) -> None: + """Control typed Supervisor observations around the real dispatcher and SQLite journal.""" + base, _base_grants, identity, signed, _uids, state = real_boundary + origin = base.oauth.canonical_issuer + oauth = OAuthSettings(base.oauth.database_path, origin, base.oauth.firebase_project_id, (OAuthClient("alice-client", (origin + "/alice-callback",), allowed_scopes=(READ_SCOPE, DEPLOY_SCOPE)),), enabled_scopes=(READ_SCOPE, DEPLOY_SCOPE)) + policy = tmp_path / "operator.yaml" + policy.write_text("profile: beta\ntargets:\n beta-worker:\n channel: beta\n repositories: [fixture/beta]\n", encoding="utf-8") + settings = ManagementSettings(tmp_path, oauth, port=base.port, operating_policy_path=policy, operation_state_root=tmp_path / "journal") + bridge = LifecycleBridge() + bridge.selected = "bootstrap" + assert bridge.worker is not None + bridge.worker["runtime_id"] = "bootstrap" + app = create_management_app(settings, OAuthGrantService(oauth, identity, clock=lambda: state["now"]), worker_clients={"beta": bridge}) # type: ignore[dict-item] + runtime = app.state.management_runtime + + async def no_background_monitor() -> None: + # Every lifecycle iteration is driven explicitly at the identity boundary. + pass + + monkeypatch.setattr(runtime, "start_monitor", no_background_monitor) + with TestClient(app, base_url=origin) as client: + bearer = _issue(client, settings, signed, "alice", "alice-client", READ_SCOPE + " " + DEPLOY_SCOPE)["access_token"] + arguments = {"target_id": "beta-worker", "idempotency_key": "owner-changes-between-observations", "expected_supervisor_generation": "supervisor-a", "expected_process_run_id": "run-old", "expected_deployment_generation": "generation-old", "expected_runtime_id": "bootstrap"} + admitted = _call(client, bearer, "auto_coder_stop_worker", arguments) + operation_id = str(admitted["operation_id"]) + effect_id = "d-" + operation_id + for _ in range(8): + _drive(app, operation_id, 1) + if effect_id in bridge.receipts: + break + else: + raise AssertionError("original drain was never dispatched") + original = runtime.journal.lifecycle_snapshot(operation_id) + original_record = runtime.journal.get(operation_id).record + assert original is not None and original[1]["stage"] == "DRAINING" + assert original[1]["steps"]["drain"]["state"] == "SENT_UNKNOWN" + original_receipt = dict(bridge.receipts[effect_id]) + actions = list(bridge.actions) + original_inspect = bridge.inspect_supervisor + inspections = 0 + + def reconstruct_between_checks() -> WorkerNetworkResult: + nonlocal inspections + inspections += 1 + observed = original_inspect() + if inspections == 1: + # The first check saw the old owner; the step sees its replacement. + bridge.supervisor_generation = "supervisor-reconstructed" + return observed + + monkeypatch.setattr(bridge, "inspect_supervisor", reconstruct_between_checks) + _drive(app, operation_id, 1) + assert inspections == 2 + assert runtime.journal.lifecycle_snapshot(operation_id) == original + assert runtime.journal.get(operation_id).record == original_record + assert bridge.actions == actions + monkeypatch.setattr(bridge, "inspect_supervisor", original_inspect) + + if receipt_state in {"pending", "expired-pending"}: + bridge.receipts[effect_id] = {**original_receipt, "status": "DRAINING"} + elif receipt_state == "digest-conflict": + bridge.receipts[effect_id] = {**original_receipt, "intent_digest": "0" * 64} + elif receipt_state == "missing": + monkeypatch.setattr(bridge, "read_worker_operation", lambda _operation: bridge._result("UNKNOWN", {})) + if receipt_state == "expired-pending": + monkeypatch.setattr("auto_coder.management.worker_lifecycle.time", SimpleNamespace(time=lambda: original[1]["deadline"] + 1)) + + _drive(app, operation_id, 8) + if receipt_state == "pending": + assert runtime.journal.lifecycle_snapshot(operation_id) == original + assert bridge.actions == actions + bridge.receipts[effect_id] = original_receipt + _drive(app, operation_id, 8) + result = _call(client, bearer, "auto_coder_operation_get", {"target_id": "beta-worker", "operation_id": operation_id}) + final = runtime.journal.lifecycle_snapshot(operation_id) + assert final is not None + assert final[1]["deadline"] == original[1]["deadline"] + assert final[1]["original_deadline"] == original[1]["original_deadline"] + assert all(final[1]["steps"]["drain"][field] == original[1]["steps"]["drain"][field] for field in ("operation_id", "sent_at", "owner_generation", "attempt_id", "correlation_identity", "expected_digest")) + assert bridge.actions == actions == [("REQUEST_GRACEFUL_STOP", effect_id)] + if receipt_state in {"complete", "pending"}: + assert result["status"] == "SUCCEEDED", result + assert final[1]["current_supervisor_generation"] == "supervisor-reconstructed" + else: + assert result["status"] == "NEEDS_ATTENTION", result + assert result["reason"] == "original_supervisor_incarnation_changed" + assert _call(client, bearer, "auto_coder_stop_worker", {**arguments, "idempotency_key": "competing-after-owner-change"})["status"] == "BUSY" + assert _call(client, bearer, "auto_coder_stop_worker", arguments)["operation_id"] == operation_id + + def test_full_policy_allows_release_process_restart_but_never_beta_source_update(real_boundary, tmp_path: Path) -> None: base, _base_grants, identity, signed, _uids, state = real_boundary origin = base.oauth.canonical_issuer From 65c2ac85572bf6aae49b3a87eebbd07a8add4bba Mon Sep 17 00:00:00 2001 From: tetsuo Date: Fri, 13 Nov 2026 09:19:11 +0900 Subject: [PATCH 24/24] Stabilize recovery VM housekeeping before route approval --- .../management-recovery-epoch.md | 21 ++-- scripts/recovery_vm/README.md | 49 ++++++--- scripts/recovery_vm/guest.sh | 32 +++--- tests/test_management_recovery_vm.py | 99 ++++++++++++------- 4 files changed, 137 insertions(+), 64 deletions(-) diff --git a/docs/client-features/management-recovery-epoch.md b/docs/client-features/management-recovery-epoch.md index f5a27cd56..bdf131d01 100644 --- a/docs/client-features/management-recovery-epoch.md +++ b/docs/client-features/management-recovery-epoch.md @@ -231,14 +231,19 @@ succeed. A missing case, failed VM, unavailable native kernel operation or faile preservation check cannot become a passed result. The VM runner and reviewed reproduction procedure are documented in `scripts/recovery_vm/README.md`. -The fresh recovery VM stops and masks only the measured -`systemd-tmpfiles-clean.timer` / `systemd-tmpfiles-clean.service` maintenance pair -before original producers and route approval. It records the actual stopped and -inhibited systemd facts and protected mask identities in -`maintenance-profile.json`; unavailable or changing state fails preparation. -This keeps scheduled maintenance from changing its approved execution readback -during the long native grant scenario. Production catalog/digest equality and -all ordinary or quarantined containment profiles remain unchanged. +The fresh recovery VM stops and masks six explicitly named timer/service pairs +from its observed stock timer inventory: `systemd-tmpfiles-clean`, `apt-daily`, +`apt-daily-upgrade`, `dpkg-db-backup`, `e2scrub_all` and `fstrim`, each with its +exact `.timer` and `.service` name. All selected timers stop before any selected +service, before original producers and route approval. The existing stock-route +fixture retains ownership of the separate `motd-news` pair. The guest records +actual stopped and inhibited systemd facts and protected mask identities in +`maintenance-profile.json`; unavailable, failed or changing state and preexisting +mask paths fail preparation. This finite setup prevents the observed scheduled +maintenance from changing its approved execution readback during native +acceptance, including the measured midnight backup execution documented in +`scripts/recovery_vm/README.md`. Production catalog/digest equality and all +ordinary or quarantined containment profiles remain unchanged. Before recovery preparation, the fixture proves that all original A/B/C managers, proxies and cutover readers have stopped. It may then close only the empty WAL diff --git a/scripts/recovery_vm/README.md b/scripts/recovery_vm/README.md index c048e9dba..2d9f6c73a 100644 --- a/scripts/recovery_vm/README.md +++ b/scripts/recovery_vm/README.md @@ -51,19 +51,44 @@ recovery approval exists. It does not change an operational host, configure the production issuer, or establish acceptance for every stock Docker service profile or an incident with missing originals. -Before original producers or recovery approval, the guest stops and masks the -exact `systemd-tmpfiles-clean.timer` / `systemd-tmpfiles-clean.service` pair. Its -ordinary scheduled execution changed the approved service's execution readback -during the longer grant scenario. Preparation now stops the timer first, waits -for the service to stop, and uses the production systemd reader to require real -protected `/dev/null` masks, inactive state, zero service PID and no pending -daemon reload. It reobserves the inhibited facts and retains the matching result, +Before original producers or recovery approval, the guest stops and masks these +six explicitly enumerated timer/service pairs from its observed stock timer +inventory: + +| Timer | Service | +| --- | --- | +| `systemd-tmpfiles-clean.timer` | `systemd-tmpfiles-clean.service` | +| `apt-daily.timer` | `apt-daily.service` | +| `apt-daily-upgrade.timer` | `apt-daily-upgrade.service` | +| `dpkg-db-backup.timer` | `dpkg-db-backup.service` | +| `e2scrub_all.timer` | `e2scrub_all.service` | +| `fstrim.timer` | `fstrim.service` | + +The existing stock-route fixture separately inhibits `motd-news.timer` and its +service. There is no wildcard timer discovery or inhibition. + +A scheduled tmpfiles execution previously changed its approved service readback. +The [later native after-receipt-head job](https://github.com/kitamura-tetsuo/auto-coder/actions/runs/43321049334/job/131543555073) +recorded target startup at `2026-11-12 23:58:08.1399639 UTC`, after route approval, +then `dpkg-db-backup.service` starting at `2026-11-13 00:00:00.2218320 UTC` and +finishing at `00:00:00.3340939 UTC`. The issuer refused at `00:01:07.623 UTC` +in the per-unit effective-route comparison, before the selected publication +boundary. Its bounded cause metadata did not export the exact mismatching unit +name. The observed post-approval backup execution changes the full `ExecStart` +readback included in that unit's strict digest even after the service returns +inactive. The finite preparation profile also covers the other observed stock +housekeeping timers, so their calendar schedules do not determine acceptance. + +Preparation stops all six timers before waiting for any selected service to +stop. It uses the production systemd reader to require real protected +`/dev/null` masks, inactive state, zero service PIDs and no pending daemon reload. +It reobserves every inhibited fact and retains the matching result, initial/stopped facts and mask identities in `maintenance-profile.json`, with -stdout and file logging. -An unavailable unit, failed stop, preexisting mask path or changed final state -fails preparation. This finite fresh-VM setup leaves the production whole-route -catalog and digest comparisons unchanged; it never refreshes an approval after -maintenance or changes an operational recovery host. +stdout and file logging. An unavailable unit, failed state or stop, preexisting +mask path or changed final state fails preparation. This finite fresh-VM setup +leaves the production whole-route catalog and digest comparisons unchanged; it +never refreshes an approval after maintenance or changes an operational recovery +host. The guest installs the exact `uv==0.13.6` release and validates its reported release number. The complete observed `--version` banner, including its bounded build diff --git a/scripts/recovery_vm/guest.sh b/scripts/recovery_vm/guest.sh index e620c37e7..687e02edd 100644 --- a/scripts/recovery_vm/guest.sh +++ b/scripts/recovery_vm/guest.sh @@ -160,13 +160,23 @@ import sys from auto_coder.logger_config import get_logger, setup_logger from auto_coder.management.containment_route_manifest import _systemd_unit -MAINTENANCE_UNITS = ("systemd-tmpfiles-clean.timer", "systemd-tmpfiles-clean.service") +MAINTENANCE_PAIRS = ( + ("systemd-tmpfiles-clean.timer", "systemd-tmpfiles-clean.service"), + ("apt-daily.timer", "apt-daily.service"), + ("apt-daily-upgrade.timer", "apt-daily-upgrade.service"), + ("dpkg-db-backup.timer", "dpkg-db-backup.service"), + ("e2scrub_all.timer", "e2scrub_all.service"), + ("fstrim.timer", "fstrim.service"), +) +MAINTENANCE_TIMERS = tuple(timer for timer, _ in MAINTENANCE_PAIRS) +MAINTENANCE_SERVICES = tuple(service for _, service in MAINTENANCE_PAIRS) +MAINTENANCE_UNITS = tuple(unit for pair in MAINTENANCE_PAIRS for unit in pair) MASK_ROOT = Path("/etc/systemd/system") logger = get_logger(__name__) def inhibit_maintenance(output): - """Settle only this fresh VM's measured maintenance pair before originals.""" + """Stop and mask this fresh VM's selected maintenance pairs before originals.""" evidence = {"schema": "auto-coder-recovery-vm-maintenance-profile-v1", "status": "preparing"} try: masks = tuple(MASK_ROOT / unit for unit in MAINTENANCE_UNITS) @@ -175,31 +185,31 @@ def inhibit_maintenance(output): initial = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) evidence["initial"] = [asdict(fact) for fact in initial] if any(not fact.unit_file_state or fact.active_state not in {"active", "inactive"} or fact.needs_daemon_reload for fact in initial): - raise ValueError("The original installed maintenance pair is unavailable") - # Stop the timer first, then wait for its service to stop. Never reset - # failed units or change an already approved recovery environment. - for unit in MAINTENANCE_UNITS: + raise ValueError("The original installed maintenance units are unavailable") + # Stop every selected timer before waiting for any selected service. + # Never reset failed units or change an approved recovery environment. + for unit in (*MAINTENANCE_TIMERS, *MAINTENANCE_SERVICES): subprocess.run(("systemctl", "stop", "--", unit), check=True, timeout=30) stopped = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) evidence["stopped"] = [asdict(fact) for fact in stopped] if any(fact.active_state != "inactive" or fact.main_pid != 0 or fact.needs_daemon_reload for fact in stopped): - raise ValueError("The actual maintenance pair did not stop") + raise ValueError("The actual maintenance units did not stop") subprocess.run(("systemctl", "mask", "--", *MAINTENANCE_UNITS), check=True, timeout=30) subprocess.run(("systemctl", "daemon-reload"), check=True, timeout=30) observed = tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) evidence["observed"] = [asdict(fact) for fact in observed] if any(not fact.inhibited or fact.unit_file_state != "masked" or fact.active_state != "inactive" or fact.main_pid != 0 or fact.needs_daemon_reload for fact in observed): - raise ValueError("The actual maintenance pair is not masked and inactive") + raise ValueError("The actual maintenance units are not masked and inactive") evidence["masks"] = [] for path in masks: info = path.lstat() if not path.is_symlink() or os.readlink(path) != "/dev/null": - raise ValueError("The maintenance pair lacks its actual /dev/null masks") + raise ValueError("The maintenance units lack their actual /dev/null masks") evidence["masks"].append({"unit": path.name, "path": str(path), "device": info.st_dev, "inode": info.st_ino, "ctime_ns": info.st_ctime_ns, "mode": info.st_mode, "uid": info.st_uid, "target": os.readlink(path)}) if tuple(_systemd_unit(unit) for unit in MAINTENANCE_UNITS) != observed: - raise ValueError("The inhibited maintenance pair changed during verification") + raise ValueError("The inhibited maintenance units changed during verification") evidence["status"] = "inhibited" - logger.info("Verified the fixed maintenance timer/service pair is masked and inactive before original fixture creation") + logger.info("Verified the fixed maintenance timer/service pairs are masked and inactive before original fixture creation") except Exception as error: evidence.update(status="failed", failure_type=type(error).__name__) logger.error("Fresh recovery VM maintenance preparation failed: {}", type(error).__name__) diff --git a/tests/test_management_recovery_vm.py b/tests/test_management_recovery_vm.py index 654569f39..3115ef4ea 100644 --- a/tests/test_management_recovery_vm.py +++ b/tests/test_management_recovery_vm.py @@ -14,17 +14,26 @@ GUEST = Path(__file__).resolve().parents[1] / "scripts/recovery_vm/guest.sh" MAINTENANCE_COMMAND = ".venv/bin/python - \"$output/maintenance-profile.json\" <<'PY'\n" -TIMER = "systemd-tmpfiles-clean.timer" -SERVICE = "systemd-tmpfiles-clean.service" -UNITS = (TIMER, SERVICE) +PAIRS = ( + ("systemd-tmpfiles-clean.timer", "systemd-tmpfiles-clean.service"), + ("apt-daily.timer", "apt-daily.service"), + ("apt-daily-upgrade.timer", "apt-daily-upgrade.service"), + ("dpkg-db-backup.timer", "dpkg-db-backup.service"), + ("e2scrub_all.timer", "e2scrub_all.service"), + ("fstrim.timer", "fstrim.service"), +) +TIMERS = tuple(timer for timer, _ in PAIRS) +SERVICES = tuple(service for _, service in PAIRS) +UNITS = tuple(unit for pair in PAIRS for unit in pair) class SystemdFixture: """Supply systemctl readback while retaining real parser and mask objects.""" - def __init__(self, root: Path, fault: str | None = None) -> None: + def __init__(self, root: Path, fault: str | None = None, fault_unit: str = SERVICES[0]) -> None: self.root = root self.fault = fault + self.fault_unit = fault_unit self.calls: list[tuple[str, ...]] = [] self.stopped: set[str] = set() self.masked = False @@ -34,37 +43,42 @@ def __init__(self, root: Path, fault: str | None = None) -> None: def command(self, *arguments: str) -> str: assert arguments[0] == "systemctl" and arguments[-1] in UNITS unit = arguments[-1] + service = unit in SERVICES if arguments[1] == "cat": - return "# Installed maintenance unit\n[Service]\nExecStart=/usr/bin/systemd-tmpfiles --clean\n" + return "# Installed maintenance unit\n" + ("[Service]\nExecStart=/usr/bin/true\n" if service else "[Timer]\nOnCalendar=daily\n") assert arguments[1] == "show" state = { "Id": unit, "LoadState": "masked" if self.masked else "loaded", "UnitFileState": "masked" if self.masked else "static", "ActiveState": "inactive" if unit in self.stopped else "active", - "SubState": "dead" if unit in self.stopped else "running" if unit == SERVICE else "waiting", + "SubState": "dead" if unit in self.stopped else "running" if service else "waiting", "NeedDaemonReload": "no", "FragmentPath": str(self.root / unit) if self.masked else "/usr/lib/systemd/system/" + unit, } - if unit == SERVICE: + if service: state["MainPID"] = "0" if unit in self.stopped else "123" - if not self.masked and self.fault == "missing" and unit == SERVICE: - state.update(LoadState="not-found", UnitFileState="", ActiveState="inactive", FragmentPath="") - if not self.masked and unit in self.stopped and self.fault == "still-running" and unit == SERVICE: - state.update(ActiveState="active", SubState="running", MainPID="123") + if unit == self.fault_unit: + if not self.masked and unit not in self.stopped and self.fault and self.fault.startswith("initial:"): + _, key, value = self.fault.split(":", 2) + state[key] = value + if not self.masked and self.fault == "missing": + state.update(LoadState="not-found", UnitFileState="", ActiveState="inactive", FragmentPath="") + if not self.masked and unit in self.stopped and self.fault == "still-running": + state.update(ActiveState="active", SubState="running", MainPID="123") if self.masked: self.reads[unit] += 1 - if unit == SERVICE and self.fault and self.fault.startswith("property:"): + if unit == self.fault_unit and self.fault and self.fault.startswith("property:"): _, key, value = self.fault.split(":", 2) if value == "missing": state.pop(key) else: state[key] = value - if unit == TIMER and self.fault == "changed-mask" and self.reads[unit] == 2: + if unit == self.fault_unit and self.fault == "changed-mask" and self.reads[unit] == 2: path = self.root / unit path.rename(self.root / "retained-original-mask") path.symlink_to("/dev/null") - if unit == TIMER: + if not service: self.timer_rows.append(state.copy()) return "\n".join(key + "=" + value for key, value in state.items()) @@ -74,17 +88,19 @@ def run(self, command: tuple[str, ...], *, check: bool, timeout: int) -> subproc if command[:3] == ("systemctl", "stop", "--"): unit = command[3] assert unit in UNITS - if self.fault == "stop-failed" and unit == SERVICE: + if unit in SERVICES: + assert set(TIMERS) <= self.stopped, "Every selected timer must stop before any service" + if self.fault == "stop-failed" and unit == self.fault_unit: raise subprocess.CalledProcessError(1, command) self.stopped.add(unit) elif command == ("systemctl", "mask", "--", *UNITS): assert self.stopped == set(UNITS) for unit in UNITS: path = self.root / unit - if self.fault == "regular-mask" and unit == SERVICE: + if self.fault == "regular-mask" and unit == self.fault_unit: path.write_text("not a mask") else: - path.symlink_to("/dev/zero" if self.fault == "wrong-mask" and unit == SERVICE else "/dev/null") + path.symlink_to("/dev/zero" if self.fault == "wrong-mask" and unit == self.fault_unit else "/dev/null") self.masked = True else: assert command == ("systemctl", "daemon-reload") and self.masked @@ -104,8 +120,8 @@ def maintenance(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): actual_reader = routes._systemd_unit namespace["_systemd_unit"] = lambda unit: actual_reader(unit, trusted_uid=os.geteuid()) - def prepare(fault: str | None = None) -> SystemdFixture: - fixture = SystemdFixture(mask_root, fault) + def prepare(fault: str | None = None, fault_unit: str = SERVICES[0]) -> SystemdFixture: + fixture = SystemdFixture(mask_root, fault, fault_unit) monkeypatch.setattr(routes, "_command", fixture.command) namespace["subprocess"] = SimpleNamespace(run=fixture.run) return fixture @@ -113,7 +129,7 @@ def prepare(fault: str | None = None) -> SystemdFixture: return namespace["inhibit_maintenance"], prepare, tmp_path / "maintenance-profile.json", mask_root -def test_guest_inhibits_only_exact_pair_before_originals_and_control_copy() -> None: +def test_guest_inhibits_only_exact_pairs_before_originals_and_control_copy() -> None: text = GUEST.read_text() setup = text.index(MAINTENANCE_COMMAND) assert text.index("verify_installed_watcher\n") < setup @@ -124,24 +140,26 @@ def test_guest_inhibits_only_exact_pair_before_originals_and_control_copy() -> N subprocess.run(("bash", "-n", str(GUEST)), check=True, timeout=10) -def test_real_guest_block_stops_timer_first_and_retains_actual_masks(maintenance) -> None: +def test_real_guest_block_stops_all_timers_before_services_and_retains_actual_masks(maintenance) -> None: inhibit, prepare, output, masks = maintenance systemd = prepare() inhibit(output) assert systemd.calls == [ - ("systemctl", "stop", "--", TIMER), - ("systemctl", "stop", "--", SERVICE), + *(("systemctl", "stop", "--", unit) for unit in (*TIMERS, *SERVICES)), ("systemctl", "mask", "--", *UNITS), ("systemctl", "daemon-reload"), ] evidence = json.loads(output.read_text()) assert evidence["schema"] == "auto-coder-recovery-vm-maintenance-profile-v1" and evidence["status"] == "inhibited" + assert [fact["unit"] for fact in evidence["initial"]] == list(UNITS) + assert [fact["unit"] for fact in evidence["stopped"]] == list(UNITS) assert [fact["unit"] for fact in evidence["observed"]] == list(UNITS) - assert evidence["initial"][1]["main_pid"] == 123 + assert all(fact["main_pid"] == 123 for fact in evidence["initial"] if fact["unit"] in SERVICES) assert all(fact["active_state"] == "inactive" and fact["main_pid"] == 0 for fact in evidence["stopped"]) assert all(fact["inhibited"] and fact["unit_file_state"] == "masked" and fact["main_pid"] == 0 and not fact["needs_daemon_reload"] for fact in evidence["observed"]) assert all("MainPID" not in row for row in systemd.timer_rows) - assert systemd.reads == {TIMER: 2, SERVICE: 2} + assert systemd.reads == dict.fromkeys(UNITS, 2) + assert [record["unit"] for record in evidence["masks"]] == list(UNITS) for record in evidence["masks"]: path = masks / record["unit"] info = path.lstat() @@ -149,9 +167,10 @@ def test_real_guest_block_stops_timer_first_and_retains_actual_masks(maintenance assert os.readlink(path) == "/dev/null" -def test_existing_mask_is_preserved_without_stopping_or_resetting(maintenance) -> None: +@pytest.mark.parametrize("unit", UNITS) +def test_existing_mask_is_preserved_without_stopping_or_resetting(maintenance, unit: str) -> None: inhibit, prepare, output, masks = maintenance - path = masks / TIMER + path = masks / unit path.symlink_to("/dev/null") before = path.lstat() systemd = prepare() @@ -162,10 +181,22 @@ def test_existing_mask_is_preserved_without_stopping_or_resetting(maintenance) - assert json.loads(output.read_text())["status"] == "failed" +@pytest.mark.parametrize("unit", UNITS) +@pytest.mark.parametrize("fault", ("initial:ActiveState:failed", "initial:NeedDaemonReload:yes")) +def test_unavailable_initial_unit_refuses_before_any_preparation(maintenance, unit: str, fault: str) -> None: + inhibit, prepare, output, masks = maintenance + systemd = prepare(fault, unit) + with pytest.raises(ValueError, match="installed maintenance"): + inhibit(output) + assert systemd.calls == [] and not tuple(masks.iterdir()) + assert json.loads(output.read_text())["status"] == "failed" + + +@pytest.mark.parametrize("service", SERVICES) @pytest.mark.parametrize("fault", ("missing", "stop-failed", "still-running")) -def test_missing_or_unstopped_service_never_installs_a_mask(maintenance, fault: str) -> None: +def test_missing_or_unstopped_service_never_installs_a_mask(maintenance, service: str, fault: str) -> None: inhibit, prepare, output, masks = maintenance - systemd = prepare(fault) + systemd = prepare(fault, service) with pytest.raises((ValueError, subprocess.CalledProcessError)): inhibit(output) assert not tuple(masks.iterdir()) @@ -177,6 +208,7 @@ def test_missing_or_unstopped_service_never_installs_a_mask(maintenance, fault: assert "observed" not in evidence +@pytest.mark.parametrize("pair", PAIRS, ids=lambda pair: pair[0].removesuffix(".timer")) @pytest.mark.parametrize( "fault", ( @@ -192,14 +224,15 @@ def test_missing_or_unstopped_service_never_installs_a_mask(maintenance, fault: "changed-mask", ), ) -def test_guest_refuses_incomplete_or_changed_actual_inhibition(maintenance, fault: str) -> None: +def test_guest_refuses_incomplete_or_changed_actual_inhibition(maintenance, pair: tuple[str, str], fault: str) -> None: inhibit, prepare, output, masks = maintenance - systemd = prepare(fault) + timer, service = pair + systemd = prepare(fault, timer if fault == "changed-mask" else service) with pytest.raises(ValueError): inhibit(output) evidence = json.loads(output.read_text()) assert evidence["status"] == "failed" and evidence["failure_type"] assert systemd.calls[-1] == ("systemctl", "daemon-reload") - assert len(systemd.calls) == 4 - assert (masks / TIMER).is_symlink() + assert len(systemd.calls) == len(UNITS) + 2 + assert (masks / timer).is_symlink() assert all("reset-failed" not in command and "unmask" not in command for command in systemd.calls)